diff --git a/.gitattributes b/.gitattributes index 28577f0416eb04985466336e4db15180d966f08c..f8dc4ebb70b3940ed2f655c63cdd308cc8c20243 100644 --- a/.gitattributes +++ b/.gitattributes @@ -150,3 +150,47 @@ yxg6asez/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text yxg6asez/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text yxg6asez/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text yxg6asez/episode_rosbags/episode_1_2025-01-04-20-49-45.bag filter=lfs diff=lfs merge=lfs -text +z9oszy8k/episode_rosbags/episode_4_2025-01-03-20-08-24.bag filter=lfs diff=lfs merge=lfs -text +tmmu86ea/episode_rosbags/episode_4_2025-01-03-20-13-33.bag filter=lfs diff=lfs merge=lfs -text +yxg6asez/episode_rosbags/episode_0_2025-01-04-20-49-05.bag filter=lfs diff=lfs merge=lfs -text +tmmu86ea/episode_rosbags/episode_3_2025-01-03-20-12-43.bag filter=lfs diff=lfs merge=lfs -text +tmmu86ea/episode_rosbags/episode_1_2025-01-03-20-11-06.bag filter=lfs diff=lfs merge=lfs -text +pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval/1_eval_1_b53855a2640f91b72a28.mp4 filter=lfs diff=lfs merge=lfs -text +pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval/0_eval_0_4895a45f2099a6358d9b.mp4 filter=lfs diff=lfs merge=lfs -text +pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval/2_eval_2_0a282a3c65bafd15a5fd.mp4 filter=lfs diff=lfs merge=lfs -text +pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval/4_eval_4_9f3ac500e502283fe4f2.mp4 filter=lfs diff=lfs merge=lfs -text +pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval/3_eval_3_20a3393f5cad2132e7ed.mp4 filter=lfs diff=lfs merge=lfs -text +yxg6asez/episode_rosbags/episode_3_2025-01-04-20-51-08.bag filter=lfs diff=lfs merge=lfs -text +pauhv4zk/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +pauhv4zk/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +pauhv4zk/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +pauhv4zk/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +pauhv4zk/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +tmmu86ea/episode_rosbags/episode_0_2025-01-03-20-10-01.bag filter=lfs diff=lfs merge=lfs -text +ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval/0_eval_0_3d045a193d2f057ea903.mp4 filter=lfs diff=lfs merge=lfs -text +ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval/4_eval_4_08c14d809d4b1c164165.mp4 filter=lfs diff=lfs merge=lfs -text +ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval/3_eval_3_506aff901ce34ad3ca80.mp4 filter=lfs diff=lfs merge=lfs -text +ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval/1_eval_1_2748fe54b33b495c534c.mp4 filter=lfs diff=lfs merge=lfs -text +ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval/2_eval_2_dd8753b9ffd389721aa4.mp4 filter=lfs diff=lfs merge=lfs -text +u0i99mq0/episode_rosbags/episode_1_2025-01-04-22-41-37.bag filter=lfs diff=lfs merge=lfs -text +u0i99mq0/episode_rosbags/episode_0_2025-01-04-22-40-57.bag filter=lfs diff=lfs merge=lfs -text +u0i99mq0/episode_rosbags/episode_2_2025-01-04-22-42-18.bag filter=lfs diff=lfs merge=lfs -text +ycy7lipi/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +ycy7lipi/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +ycy7lipi/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +ycy7lipi/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +ycy7lipi/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +u0i99mq0/episode_rosbags/episode_3_2025-01-04-22-49-33.bag filter=lfs diff=lfs merge=lfs -text +u0i99mq0/episode_rosbags/episode_4_2025-01-04-22-50-18.bag filter=lfs diff=lfs merge=lfs -text +chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval/3_eval_3_c975888550477f48ba44.mp4 filter=lfs diff=lfs merge=lfs -text +chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval/2_eval_2_8a57ace6d7baf7108296.mp4 filter=lfs diff=lfs merge=lfs -text +chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval/4_eval_4_d357aac167e5d7bc548b.mp4 filter=lfs diff=lfs merge=lfs -text +amw07z8o/episode_rosbags/episode_3_2025-01-04-22-38-18.bag filter=lfs diff=lfs merge=lfs -text +chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval/0_eval_0_46f66c2ba3db8f12f00c.mp4 filter=lfs diff=lfs merge=lfs -text +chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval/1_eval_1_b0947b11fcc3d3b55612.mp4 filter=lfs diff=lfs merge=lfs -text +chnx0pdg/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +amw07z8o/episode_rosbags/episode_1_2025-01-04-22-36-27.bag filter=lfs diff=lfs merge=lfs -text +chnx0pdg/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +chnx0pdg/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +chnx0pdg/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +chnx0pdg/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text diff --git a/amw07z8o/episode_rosbags/episode_1_2025-01-04-22-36-27.bag b/amw07z8o/episode_rosbags/episode_1_2025-01-04-22-36-27.bag new file mode 100644 index 0000000000000000000000000000000000000000..7337d5c86c568308b5dc76a549275e0103f9c075 --- /dev/null +++ b/amw07z8o/episode_rosbags/episode_1_2025-01-04-22-36-27.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ec6b1bcb00afd78e4c1ec8a7c28eee5e0fd7114fb7e10cbe40a92174c1c149f +size 1412723535 diff --git a/amw07z8o/episode_rosbags/episode_3_2025-01-04-22-38-18.bag b/amw07z8o/episode_rosbags/episode_3_2025-01-04-22-38-18.bag new file mode 100644 index 0000000000000000000000000000000000000000..8eb50feb118ff4858dd50938929a1a707339ee64 --- /dev/null +++ b/amw07z8o/episode_rosbags/episode_3_2025-01-04-22-38-18.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f065240e2bd8c3374c819edaccafef829620b35b83e311daddf821ec50772450 +size 1410539711 diff --git a/c8gy6khn/.hydra/config.yaml b/c8gy6khn/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..30c3bfdc0a6aec7938f3f2b6ed72878e72cc1f21 --- /dev/null +++ b/c8gy6khn/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1938_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/c8gy6khn/.hydra/hydra.yaml b/c8gy6khn/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e6948df432dd45386a4b5d397c1405da5fe19c9c --- /dev/null +++ b/c8gy6khn/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/c8gy6khn + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/c8gy6khn/.hydra/overrides.yaml b/c8gy6khn/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/c8gy6khn/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/c8gy6khn/eval_policy.log b/c8gy6khn/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..5808f1a028af3db8f20748f2643919cb2c0d16a0 --- /dev/null +++ b/c8gy6khn/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-04 20:48:39,452][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-04 20:48:39,455][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-04 20:48:43,186][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/chnx0pdg/.hydra/config.yaml b/chnx0pdg/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..30c3bfdc0a6aec7938f3f2b6ed72878e72cc1f21 --- /dev/null +++ b/chnx0pdg/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1938_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/chnx0pdg/.hydra/hydra.yaml b/chnx0pdg/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..efe7fb6f7f5b8ff60a936c37d2410942af399f62 --- /dev/null +++ b/chnx0pdg/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/chnx0pdg/.hydra/overrides.yaml b/chnx0pdg/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/chnx0pdg/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/chnx0pdg/episode_rosbags/aligned_depth_to_color_K.npy b/chnx0pdg/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/chnx0pdg/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/chnx0pdg/episode_rosbags/cam_tf_world.npy b/chnx0pdg/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/chnx0pdg/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/chnx0pdg/episode_rosbags/color_K.npy b/chnx0pdg/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/chnx0pdg/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/chnx0pdg/episode_rosbags/depth_K.npy b/chnx0pdg/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/chnx0pdg/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/chnx0pdg/eval_robot.log b/chnx0pdg/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..4f1246f6518ae171f850da9de82eb28ab0318c32 --- /dev/null +++ b/chnx0pdg/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-04 21:23:21,858][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-04 21:23:21,861][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/chnx0pdg/eval_video/0_eval.mp4 b/chnx0pdg/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..25d7a346c48cbd18adb166e216b3389c4c3f08e3 --- /dev/null +++ b/chnx0pdg/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46f66c2ba3db8f12f00c5bc7aa19ba0e5eb6f4f2c12aa8227d2e23c96c9b96b8 +size 1195059 diff --git a/chnx0pdg/eval_video/1_eval.mp4 b/chnx0pdg/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a7e88088835eb267a9aed5ec8b4b3daa0af0eb07 --- /dev/null +++ b/chnx0pdg/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0947b11fcc3d3b55612d61edb7fd1f3364879e96b5a1b96730b590cccc47ed5 +size 1193365 diff --git a/chnx0pdg/eval_video/2_eval.mp4 b/chnx0pdg/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2f670a247e39382dd7343367e65cf55b4f827f04 --- /dev/null +++ b/chnx0pdg/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a57ace6d7baf7108296c5eb6844321ff8138f5cfd494b222e13551ec8e9aa68 +size 1196574 diff --git a/chnx0pdg/eval_video/3_eval.mp4 b/chnx0pdg/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..33a9fea0d0949387d46cd21c61c0e9ab9e7be789 --- /dev/null +++ b/chnx0pdg/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c975888550477f48ba44a40bea300cfb9a0494e1be02f664ae0080151df75294 +size 1192068 diff --git a/chnx0pdg/eval_video/4_eval.mp4 b/chnx0pdg/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..65ff415da7e1478410c680067527b9b5e547ac1d --- /dev/null +++ b/chnx0pdg/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d357aac167e5d7bc548b9c4a8147292bbc6865983c746e7848689ead6f3ad4e3 +size 1202203 diff --git a/chnx0pdg/tb/events.out.tfevents.1736043808.leonmkim-ROG-Strix-G15CS-G15CS.2025958.0 b/chnx0pdg/tb/events.out.tfevents.1736043808.leonmkim-ROG-Strix-G15CS-G15CS.2025958.0 new file mode 100644 index 0000000000000000000000000000000000000000..f9aded50a78c69c033b8dfe312c0a6e864bdb107 --- /dev/null +++ b/chnx0pdg/tb/events.out.tfevents.1736043808.leonmkim-ROG-Strix-G15CS-G15CS.2025958.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80468ac0df38caf138c29bd8be079d24650a601d70399613ead5b716c45bbb99 +size 6539 diff --git a/chnx0pdg/wandb/debug-internal.log b/chnx0pdg/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..b74024d555f04fd296f4d314b8b2e8db17c7a3b0 --- /dev/null +++ b/chnx0pdg/wandb/debug-internal.log @@ -0,0 +1,784 @@ +2025-01-04 21:23:27,663 INFO StreamThr :2026123 [internal.py:wandb_internal():85] W&B internal server running at pid: 2026123, started at: 2025-01-04 21:23:27.662939 +2025-01-04 21:23:27,664 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status +2025-01-04 21:23:27,675 INFO WriterThread:2026123 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/run-chnx0pdg.wandb +2025-01-04 21:23:27,676 DEBUG SenderThread:2026123 [sender.py:send():379] send: header +2025-01-04 21:23:27,685 DEBUG SenderThread:2026123 [sender.py:send():379] send: run +2025-01-04 21:23:27,927 INFO SenderThread:2026123 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files +2025-01-04 21:23:27,927 INFO SenderThread:2026123 [sender.py:_start_run_threads():1188] run started: chnx0pdg with start time 1736043807.664176 +2025-01-04 21:23:27,942 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: check_version +2025-01-04 21:23:27,943 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: check_version +2025-01-04 21:23:27,988 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: run_start +2025-01-04 21:23:27,993 DEBUG HandlerThread:2026123 [system_info.py:__init__():26] System info init +2025-01-04 21:23:27,993 DEBUG HandlerThread:2026123 [system_info.py:__init__():41] System info init done +2025-01-04 21:23:27,993 INFO HandlerThread:2026123 [system_monitor.py:start():194] Starting system monitor +2025-01-04 21:23:27,993 INFO SystemMonitor:2026123 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-04 21:23:27,994 INFO SystemMonitor:2026123 [interfaces.py:start():188] Started cpu monitoring +2025-01-04 21:23:27,994 INFO SystemMonitor:2026123 [interfaces.py:start():188] Started disk monitoring +2025-01-04 21:23:27,995 INFO HandlerThread:2026123 [system_monitor.py:probe():214] Collecting system info +2025-01-04 21:23:27,995 INFO SystemMonitor:2026123 [interfaces.py:start():188] Started gpu monitoring +2025-01-04 21:23:27,996 INFO SystemMonitor:2026123 [interfaces.py:start():188] Started memory monitoring +2025-01-04 21:23:27,996 INFO SystemMonitor:2026123 [interfaces.py:start():188] Started network monitoring +2025-01-04 21:23:28,006 DEBUG HandlerThread:2026123 [system_info.py:probe():152] Probing system +2025-01-04 21:23:28,008 DEBUG HandlerThread:2026123 [system_info.py:_probe_git():137] Probing git +2025-01-04 21:23:28,012 DEBUG HandlerThread:2026123 [system_info.py:_probe_git():145] Probing git done +2025-01-04 21:23:28,012 DEBUG HandlerThread:2026123 [system_info.py:probe():200] Probing system done +2025-01-04 21:23:28,012 DEBUG HandlerThread:2026123 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-05T02:23:28.006710', 'startedAt': '2025-01-05T02:23:27.651297', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '02cf07d901f3d8187a302cd51c9faa80321417ee'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4125.005625, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.998, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.951, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 669.7662467956543}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-04 21:23:28,012 INFO HandlerThread:2026123 [system_monitor.py:probe():224] Finished collecting system info +2025-01-04 21:23:28,012 INFO HandlerThread:2026123 [system_monitor.py:probe():227] Publishing system info +2025-01-04 21:23:28,012 DEBUG HandlerThread:2026123 [system_info.py:_save_code():44] Saving code +2025-01-04 21:23:28,014 DEBUG HandlerThread:2026123 [system_info.py:_save_code():67] Saving code done +2025-01-04 21:23:28,014 DEBUG HandlerThread:2026123 [system_info.py:_save_patches():84] Saving git patches +2025-01-04 21:23:28,078 DEBUG HandlerThread:2026123 [system_info.py:_save_patches():126] Saving git patches done +2025-01-04 21:23:28,079 INFO HandlerThread:2026123 [system_monitor.py:probe():229] Finished publishing system info +2025-01-04 21:23:28,082 DEBUG SenderThread:2026123 [sender.py:send():379] send: files +2025-01-04 21:23:28,082 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-04 21:23:28,082 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-04 21:23:28,083 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-04 21:23:28,312 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: python_packages +2025-01-04 21:23:28,312 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:23:28,313 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: python_packages +2025-01-04 21:23:28,313 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:23:28,315 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:28,317 INFO HandlerThread:2026123 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/tb" + save: true +} +control { + relay_id: "59aad6f3f1b84a2a823baf79aed48f1c" +} +_info { + stream_id: "chnx0pdg" +} + +2025-01-04 21:23:28,317 DEBUG HandlerThread:2026123 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-04 21:23:28,324 INFO wandb-upload_0:2026123 [upload_job.py:push():130] Uploaded file /tmp/tmpq2mmia7ewandb/q93gcekk-wandb-metadata.json +2025-01-04 21:23:28,362 INFO wandb-upload_2:2026123 [upload_job.py:push():130] Uploaded file /tmp/tmpq2mmia7ewandb/9npmef1b-diff.patch +2025-01-04 21:23:28,380 DEBUG SenderThread:2026123 [sender.py:send():379] send: telemetry +2025-01-04 21:23:28,381 DEBUG SenderThread:2026123 [sender.py:send():379] send: tbrecord +2025-01-04 21:23:28,381 DEBUG SenderThread:2026123 [sender.py:send():379] send: files +2025-01-04 21:23:28,381 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file events.out.tfevents.1736043808.leonmkim-ROG-Strix-G15CS-G15CS.2025958.0 with policy live +2025-01-04 21:23:28,394 INFO wandb-upload_1:2026123 [upload_job.py:push():130] Uploaded file /tmp/tmpq2mmia7ewandb/1qfyl7ip-code/FISH/eval_robot.py +2025-01-04 21:23:28,580 INFO wandb-upload_0:2026123 [upload_job.py:push():130] Uploaded file /tmp/tmpq2mmia7ewandb/27s3y4w3-events.out.tfevents.1736043808.leonmkim-ROG-Strix-G15CS-G15CS.2025958.0 +2025-01-04 21:23:28,929 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:23:28,929 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/code/FISH/eval_robot.py +2025-01-04 21:23:28,929 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/events.out.tfevents.1736043808.leonmkim-ROG-Strix-G15CS-G15CS.2025958.0 +2025-01-04 21:23:28,929 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/requirements.txt +2025-01-04 21:23:28,929 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/diff.patch +2025-01-04 21:23:28,929 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/wandb-metadata.json +2025-01-04 21:23:28,929 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/code +2025-01-04 21:23:28,929 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/code/FISH +2025-01-04 21:23:29,313 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:30,313 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:30,928 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:23:31,313 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:32,314 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:32,406 DEBUG SenderThread:2026123 [sender.py:send():379] send: config +2025-01-04 21:23:33,313 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:23:33,314 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:34,314 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:34,929 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:23:35,314 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:36,315 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:37,314 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:38,313 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:23:38,314 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:38,930 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:23:39,315 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:40,315 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:40,931 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:23:41,315 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:42,315 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:43,312 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:23:43,313 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:23:43,353 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:43,404 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:23:44,315 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:45,315 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:46,315 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:47,316 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:48,316 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:49,314 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:23:49,316 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:50,316 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:51,316 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:52,316 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:52,934 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:23:53,317 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:54,317 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:54,935 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:23:55,314 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:23:55,317 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:56,317 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:56,935 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:23:57,317 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:58,313 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:23:58,313 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:23:58,353 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:59,317 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:00,317 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:00,936 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:24:01,069 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:24:01,317 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:01,937 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/config.yaml +2025-01-04 21:24:02,317 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:02,937 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:24:03,318 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:04,318 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:05,319 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:06,314 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:24:06,320 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:07,320 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:08,320 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:09,320 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:10,320 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:11,314 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:24:11,320 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:12,321 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:13,313 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:24:13,313 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:24:13,364 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:14,321 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:15,321 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:16,321 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:16,400 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:24:17,321 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:18,321 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:19,321 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:20,322 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:21,322 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:22,314 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:24:22,322 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:23,323 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:23,327 DEBUG SenderThread:2026123 [sender.py:send():379] send: files +2025-01-04 21:24:23,327 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_46f66c2ba3db8f12f00c.mp4 with policy now +2025-01-04 21:24:23,373 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:24:23,374 DEBUG SenderThread:2026123 [sender.py:send():379] send: history +2025-01-04 21:24:23,374 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:24:23,375 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:24:23,589 INFO wandb-upload_2:2026123 [upload_job.py:push():130] Uploaded file /tmp/tmpq2mmia7ewandb/i0snrcat-media/videos/eval/0_eval_0_46f66c2ba3db8f12f00c.mp4 +2025-01-04 21:24:23,943 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/wandb-summary.json +2025-01-04 21:24:23,943 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval/0_eval_0_46f66c2ba3db8f12f00c.mp4 +2025-01-04 21:24:23,943 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos +2025-01-04 21:24:23,943 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media +2025-01-04 21:24:23,943 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval +2025-01-04 21:24:24,206 INFO wandb-upload_0:2026123 [upload_job.py:push():130] Uploaded file /tmp/tmpq2mmia7ewandb/4gjf1xvr-media/videos/eval/0_eval_0_46f66c2ba3db8f12f00c.mp4 +2025-01-04 21:24:24,322 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:24,943 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:24:25,323 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:26,323 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:27,323 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:27,989 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:24:27,996 DEBUG SystemMonitor:2026123 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-04 21:24:27,997 DEBUG SenderThread:2026123 [sender.py:send():379] send: stats +2025-01-04 21:24:28,313 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:24:28,313 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:24:28,361 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:28,944 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:24:29,324 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:30,323 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:30,945 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:24:31,323 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:32,323 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:33,314 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:24:33,323 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:34,324 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:35,324 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:35,947 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/events.out.tfevents.1736043808.leonmkim-ROG-Strix-G15CS-G15CS.2025958.0 +2025-01-04 21:24:36,180 INFO wandb-upload_2:2026123 [upload_job.py:push():130] Uploaded file /tmp/tmpq2mmia7ewandb/9teuzw8i-events.out.tfevents.1736043808.leonmkim-ROG-Strix-G15CS-G15CS.2025958.0 +2025-01-04 21:24:36,324 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:36,947 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:24:37,324 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:38,314 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:24:38,324 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:38,948 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:24:39,324 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:40,324 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:40,948 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:24:41,324 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:42,324 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:43,313 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:24:43,313 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:24:43,357 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:43,471 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:24:44,327 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:45,327 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:46,327 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:47,328 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:48,328 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:49,315 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:24:49,328 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:50,330 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:51,330 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:52,330 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:53,330 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:54,315 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:24:54,331 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:55,331 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:56,331 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:57,331 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:57,997 DEBUG SenderThread:2026123 [sender.py:send():379] send: stats +2025-01-04 21:24:58,313 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:24:58,313 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:24:58,361 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:59,331 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:59,701 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:25:00,331 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:00,955 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:25:01,331 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:02,331 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:02,955 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:25:03,331 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:04,332 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:04,885 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:25:04,956 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:25:05,332 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:06,332 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:07,334 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:07,464 DEBUG SenderThread:2026123 [sender.py:send():379] send: files +2025-01-04 21:25:07,464 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_b0947b11fcc3d3b55612.mp4 with policy now +2025-01-04 21:25:07,467 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:25:07,467 DEBUG SenderThread:2026123 [sender.py:send():379] send: history +2025-01-04 21:25:07,468 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:25:07,468 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:25:07,725 INFO wandb-upload_0:2026123 [upload_job.py:push():130] Uploaded file /tmp/tmpq2mmia7ewandb/yyibyam3-media/videos/eval/1_eval_1_b0947b11fcc3d3b55612.mp4 +2025-01-04 21:25:07,957 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/wandb-summary.json +2025-01-04 21:25:07,957 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval/1_eval_1_b0947b11fcc3d3b55612.mp4 +2025-01-04 21:25:07,957 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval +2025-01-04 21:25:08,237 INFO wandb-upload_2:2026123 [upload_job.py:push():130] Uploaded file /tmp/tmpq2mmia7ewandb/sucky324-media/videos/eval/1_eval_1_b0947b11fcc3d3b55612.mp4 +2025-01-04 21:25:08,341 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:08,957 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:25:09,337 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:09,958 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:25:10,331 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:25:10,339 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:11,341 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:12,340 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:12,959 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:25:13,325 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:25:13,325 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:25:13,365 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:14,338 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:14,960 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:25:15,338 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:15,917 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:25:16,339 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:17,339 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:18,339 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:19,339 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:20,341 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:21,316 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:25:21,339 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:22,340 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:23,340 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:24,340 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:25,340 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:26,316 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:25:26,340 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:27,340 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:27,998 DEBUG SenderThread:2026123 [sender.py:send():379] send: stats +2025-01-04 21:25:28,314 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:25:28,314 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:25:28,357 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:29,341 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:30,344 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:31,341 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:31,456 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:25:32,341 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:33,341 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:34,341 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:35,342 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:36,342 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:37,317 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:25:37,345 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:38,345 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:39,346 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:40,344 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:41,345 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:42,317 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:25:42,345 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:43,314 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:25:43,314 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:25:43,361 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:44,345 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:44,760 DEBUG SenderThread:2026123 [sender.py:send():379] send: history +2025-01-04 21:25:44,760 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:25:44,760 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:25:44,969 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/events.out.tfevents.1736043808.leonmkim-ROG-Strix-G15CS-G15CS.2025958.0 +2025-01-04 21:25:44,970 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/wandb-summary.json +2025-01-04 21:25:45,256 INFO wandb-upload_0:2026123 [upload_job.py:push():130] Uploaded file /tmp/tmpq2mmia7ewandb/9905gted-events.out.tfevents.1736043808.leonmkim-ROG-Strix-G15CS-G15CS.2025958.0 +2025-01-04 21:25:45,346 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:46,345 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:46,970 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:25:47,345 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:47,577 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:25:48,345 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:49,346 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:50,346 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:50,971 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:25:51,346 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:52,346 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:53,317 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:25:53,346 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:54,346 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:55,346 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:56,346 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:57,346 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:57,998 DEBUG SenderThread:2026123 [sender.py:send():379] send: stats +2025-01-04 21:25:58,314 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:25:58,314 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:25:58,361 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:58,473 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:25:59,346 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:00,347 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:01,347 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:02,347 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:03,347 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:04,317 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:26:04,347 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:05,347 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:06,347 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:07,347 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:08,347 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:09,318 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:26:09,347 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:10,347 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:11,348 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:12,350 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:13,314 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:26:13,314 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:26:13,357 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:14,350 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:14,477 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:26:15,352 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:16,353 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:16,711 DEBUG SenderThread:2026123 [sender.py:send():379] send: files +2025-01-04 21:26:16,711 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_8a57ace6d7baf7108296.mp4 with policy now +2025-01-04 21:26:16,717 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:26:16,717 DEBUG SenderThread:2026123 [sender.py:send():379] send: history +2025-01-04 21:26:16,717 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:26:16,717 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:26:16,980 INFO wandb-upload_2:2026123 [upload_job.py:push():130] Uploaded file /tmp/tmpq2mmia7ewandb/ez8zh6vy-media/videos/eval/2_eval_2_8a57ace6d7baf7108296.mp4 +2025-01-04 21:26:16,981 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:26:16,981 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/wandb-summary.json +2025-01-04 21:26:16,981 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval/2_eval_2_8a57ace6d7baf7108296.mp4 +2025-01-04 21:26:16,981 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval +2025-01-04 21:26:17,281 INFO wandb-upload_0:2026123 [upload_job.py:push():130] Uploaded file /tmp/tmpq2mmia7ewandb/wwneybbg-media/videos/eval/2_eval_2_8a57ace6d7baf7108296.mp4 +2025-01-04 21:26:17,355 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:17,981 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:26:18,355 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:18,982 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:26:19,365 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:20,319 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:26:20,381 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:21,369 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:22,361 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:22,983 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:26:23,358 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:24,361 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:24,984 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:26:25,368 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:26,319 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:26:26,360 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:26,590 DEBUG SenderThread:2026123 [sender.py:send():379] send: history +2025-01-04 21:26:26,590 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:26:26,590 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:26:26,985 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/events.out.tfevents.1736043808.leonmkim-ROG-Strix-G15CS-G15CS.2025958.0 +2025-01-04 21:26:26,985 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/wandb-summary.json +2025-01-04 21:26:27,372 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:27,999 DEBUG SenderThread:2026123 [sender.py:send():379] send: stats +2025-01-04 21:26:28,325 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:26:28,325 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:26:28,365 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:28,986 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:26:29,370 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:30,397 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:31,394 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:31,672 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:26:32,393 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:32,987 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:26:33,394 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:34,394 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:35,394 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:36,394 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:37,394 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:37,663 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:26:38,395 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:38,989 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:26:39,395 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:40,397 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:41,397 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:42,397 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:43,317 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:26:43,317 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:26:43,397 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:43,454 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:26:44,397 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:45,397 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:46,397 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:47,397 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:48,397 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:49,319 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:26:49,398 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:50,398 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:51,398 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:52,398 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:53,398 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:54,398 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:55,319 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:26:55,398 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:56,398 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:57,433 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:58,000 DEBUG SenderThread:2026123 [sender.py:send():379] send: stats +2025-01-04 21:26:58,326 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:26:58,326 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:26:58,400 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:58,961 DEBUG SenderThread:2026123 [sender.py:send():379] send: files +2025-01-04 21:26:58,961 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_c975888550477f48ba44.mp4 with policy now +2025-01-04 21:26:58,965 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:26:58,966 DEBUG SenderThread:2026123 [sender.py:send():379] send: history +2025-01-04 21:26:58,966 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:26:58,966 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:26:58,998 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:26:58,998 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/wandb-summary.json +2025-01-04 21:26:58,998 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval/3_eval_3_c975888550477f48ba44.mp4 +2025-01-04 21:26:58,999 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval +2025-01-04 21:26:59,222 INFO wandb-upload_2:2026123 [upload_job.py:push():130] Uploaded file /tmp/tmpq2mmia7ewandb/vahuxfyw-media/videos/eval/3_eval_3_c975888550477f48ba44.mp4 +2025-01-04 21:26:59,399 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:59,494 INFO wandb-upload_0:2026123 [upload_job.py:push():130] Uploaded file /tmp/tmpq2mmia7ewandb/856h6i6a-media/videos/eval/3_eval_3_c975888550477f48ba44.mp4 +2025-01-04 21:26:59,998 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:27:00,425 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:00,998 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:27:00,999 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:27:01,406 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:01,999 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:27:02,407 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:03,409 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:04,407 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:05,000 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:27:05,410 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:06,410 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:06,509 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:27:07,001 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:27:07,411 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:08,412 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:09,410 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:10,003 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/events.out.tfevents.1736043808.leonmkim-ROG-Strix-G15CS-G15CS.2025958.0 +2025-01-04 21:27:10,241 INFO wandb-upload_2:2026123 [upload_job.py:push():130] Uploaded file /tmp/tmpq2mmia7ewandb/kehe1lgk-events.out.tfevents.1736043808.leonmkim-ROG-Strix-G15CS-G15CS.2025958.0 +2025-01-04 21:27:10,412 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:10,641 DEBUG SenderThread:2026123 [sender.py:send():379] send: history +2025-01-04 21:27:10,641 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:27:10,641 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:27:11,003 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:27:11,004 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/wandb-summary.json +2025-01-04 21:27:11,428 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:11,805 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:27:12,431 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:13,004 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:27:13,334 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:27:13,334 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:27:13,416 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:14,415 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:15,005 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:27:15,415 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:16,415 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:16,831 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:27:17,415 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:18,415 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:19,415 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:20,415 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:21,416 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:22,330 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:27:22,416 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:23,416 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:24,416 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:25,416 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:26,416 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:27,330 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:27:27,416 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:28,000 DEBUG SenderThread:2026123 [sender.py:send():379] send: stats +2025-01-04 21:27:28,320 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:27:28,320 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:27:28,417 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:29,416 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:30,417 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:31,417 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:32,418 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:32,419 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:27:33,418 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:34,417 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:35,417 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:36,418 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:37,418 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:38,330 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:27:38,418 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:39,418 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:40,418 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:41,421 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:41,847 DEBUG SenderThread:2026123 [sender.py:send():379] send: files +2025-01-04 21:27:41,847 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_d357aac167e5d7bc548b.mp4 with policy now +2025-01-04 21:27:41,850 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:27:41,850 DEBUG SenderThread:2026123 [sender.py:send():379] send: history +2025-01-04 21:27:41,850 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:27:41,850 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:27:42,015 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/wandb-summary.json +2025-01-04 21:27:42,015 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval/4_eval_4_d357aac167e5d7bc548b.mp4 +2025-01-04 21:27:42,015 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval +2025-01-04 21:27:42,105 INFO wandb-upload_0:2026123 [upload_job.py:push():130] Uploaded file /tmp/tmpq2mmia7ewandb/qycayt6k-media/videos/eval/4_eval_4_d357aac167e5d7bc548b.mp4 +2025-01-04 21:27:42,369 INFO wandb-upload_2:2026123 [upload_job.py:push():130] Uploaded file /tmp/tmpq2mmia7ewandb/yygn0fri-media/videos/eval/4_eval_4_d357aac167e5d7bc548b.mp4 +2025-01-04 21:27:42,423 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:43,015 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:27:43,325 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:27:43,325 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:27:43,413 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:27:43,425 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:44,434 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:45,432 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:46,437 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:47,017 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:27:47,438 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:48,443 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:48,444 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:27:49,446 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:50,448 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:51,018 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:27:51,451 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:52,457 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:53,458 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:54,342 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:27:54,448 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:55,460 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:56,457 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:57,469 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:57,696 DEBUG SenderThread:2026123 [sender.py:send():379] send: history +2025-01-04 21:27:57,696 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:27:57,697 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:27:57,981 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 21:27:57,981 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 21:27:58,020 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/events.out.tfevents.1736043808.leonmkim-ROG-Strix-G15CS-G15CS.2025958.0 +2025-01-04 21:27:58,020 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/wandb-summary.json +2025-01-04 21:27:58,021 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-04 21:27:58,021 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media +2025-01-04 21:27:58,021 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/table +2025-01-04 21:27:58,021 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/table/eval +2025-01-04 21:27:58,134 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:27:58,214 INFO wandb-upload_0:2026123 [upload_job.py:push():130] Uploaded file /tmp/tmpq2mmia7ewandb/noyckbw3-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-04 21:27:58,321 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:27:58,452 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:58,466 INFO wandb-upload_2:2026123 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpltq5jpvh +2025-01-04 21:27:58,581 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 21:27:58,591 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:27:58,935 INFO SenderThread:2026123 [sender.py:send_request_log_artifact():1525] logged artifact run-chnx0pdg-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE1MTY5OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1NTY4NQ==', 'latestArtifact': None}} +2025-01-04 21:27:58,935 DEBUG SenderThread:2026123 [sender.py:send():379] send: files +2025-01-04 21:27:58,935 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-04 21:27:58,935 DEBUG SenderThread:2026123 [sender.py:send():379] send: stats +2025-01-04 21:27:58,935 DEBUG SenderThread:2026123 [sender.py:send():379] send: history +2025-01-04 21:27:58,935 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:27:58,936 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:27:58,936 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:27:59,021 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/events.out.tfevents.1736043808.leonmkim-ROG-Strix-G15CS-G15CS.2025958.0 +2025-01-04 21:27:59,021 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:27:59,021 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/wandb-summary.json +2025-01-04 21:27:59,021 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/table/eval/total_success_rate_ci_6_2ba98cbc4fea23f33f14.table.json +2025-01-04 21:27:59,021 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/table/eval +2025-01-04 21:27:59,096 DEBUG SenderThread:2026123 [sender.py:send():379] send: telemetry +2025-01-04 21:27:59,096 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:27:59,096 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:27:59,096 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:27:59,097 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:27:59,097 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:27:59,097 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:27:59,097 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:27:59,098 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:27:59,098 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:27:59,098 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:27:59,098 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:27:59,099 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:27:59,099 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:27:59,099 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:27:59,099 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:27:59,099 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:27:59,100 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:27:59,100 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:27:59,100 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:27:59,101 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:27:59,101 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:27:59,101 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:27:59,101 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 21:27:59,176 INFO wandb-upload_0:2026123 [upload_job.py:push():130] Uploaded file /tmp/tmpq2mmia7ewandb/4hhuic55-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-04 21:27:59,235 INFO wandb-upload_2:2026123 [upload_job.py:push():130] Uploaded file /tmp/tmpq2mmia7ewandb/s4ncw0ro-media/table/eval/total_success_rate_ci_6_2ba98cbc4fea23f33f14.table.json +2025-01-04 21:27:59,452 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:59,476 INFO wandb-upload_1:2026123 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpa6t6pi25 +2025-01-04 21:27:59,847 INFO SenderThread:2026123 [sender.py:send_request_log_artifact():1525] logged artifact run-chnx0pdg-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE1MTcxMw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1NTY4Nw==', 'latestArtifact': None}} +2025-01-04 21:27:59,847 DEBUG SenderThread:2026123 [sender.py:send():379] send: files +2025-01-04 21:27:59,847 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_2ba98cbc4fea23f33f14.table.json with policy now +2025-01-04 21:27:59,847 DEBUG SenderThread:2026123 [sender.py:send():379] send: history +2025-01-04 21:27:59,848 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:27:59,848 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:27:59,849 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:28:00,021 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/wandb-summary.json +2025-01-04 21:28:00,087 INFO wandb-upload_2:2026123 [upload_job.py:push():130] Uploaded file /tmp/tmpq2mmia7ewandb/6c3bxxeq-media/table/eval/total_success_rate_ci_6_2ba98cbc4fea23f33f14.table.json +2025-01-04 21:28:00,453 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:28:01,022 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:28:01,453 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:28:02,453 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:28:03,453 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:28:04,453 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:28:05,138 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:28:05,181 DEBUG SenderThread:2026123 [sender.py:send():379] send: exit +2025-01-04 21:28:05,181 INFO SenderThread:2026123 [sender.py:send_exit():586] handling exit code: 0 +2025-01-04 21:28:05,182 INFO SenderThread:2026123 [sender.py:send_exit():588] handling runtime: 277 +2025-01-04 21:28:05,182 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:28:05,182 INFO SenderThread:2026123 [sender.py:send_exit():594] send defer +2025-01-04 21:28:05,182 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:28:05,182 INFO HandlerThread:2026123 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-04 21:28:05,182 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: defer +2025-01-04 21:28:05,182 INFO SenderThread:2026123 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-04 21:28:05,182 INFO SenderThread:2026123 [sender.py:transition_state():614] send defer: 1 +2025-01-04 21:28:05,183 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:28:05,183 INFO HandlerThread:2026123 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-04 21:28:05,183 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: defer +2025-01-04 21:28:05,183 INFO SenderThread:2026123 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-04 21:28:05,183 INFO SenderThread:2026123 [sender.py:transition_state():614] send defer: 2 +2025-01-04 21:28:05,183 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:28:05,183 INFO HandlerThread:2026123 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-04 21:28:05,183 INFO HandlerThread:2026123 [system_monitor.py:finish():203] Stopping system monitor +2025-01-04 21:28:05,183 DEBUG SystemMonitor:2026123 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-04 21:28:05,183 DEBUG SystemMonitor:2026123 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-04 21:28:05,184 INFO HandlerThread:2026123 [interfaces.py:finish():200] Joined cpu monitor +2025-01-04 21:28:05,184 INFO HandlerThread:2026123 [interfaces.py:finish():200] Joined disk monitor +2025-01-04 21:28:05,237 INFO HandlerThread:2026123 [interfaces.py:finish():200] Joined gpu monitor +2025-01-04 21:28:05,237 INFO HandlerThread:2026123 [interfaces.py:finish():200] Joined memory monitor +2025-01-04 21:28:05,238 INFO HandlerThread:2026123 [interfaces.py:finish():200] Joined network monitor +2025-01-04 21:28:05,238 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: defer +2025-01-04 21:28:05,238 INFO SenderThread:2026123 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-04 21:28:05,238 INFO SenderThread:2026123 [sender.py:transition_state():614] send defer: 3 +2025-01-04 21:28:05,238 DEBUG SenderThread:2026123 [sender.py:send():379] send: stats +2025-01-04 21:28:05,238 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:28:05,238 INFO HandlerThread:2026123 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-04 21:28:05,239 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: defer +2025-01-04 21:28:05,239 INFO SenderThread:2026123 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-04 21:28:05,239 INFO SenderThread:2026123 [sender.py:transition_state():614] send defer: 4 +2025-01-04 21:28:05,239 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:28:05,239 INFO HandlerThread:2026123 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-04 21:28:06,024 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/wandb-summary.json +2025-01-04 21:28:07,024 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:28:11,702 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:28:11,703 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: defer +2025-01-04 21:28:11,703 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:28:11,703 INFO SenderThread:2026123 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-04 21:28:11,703 INFO SenderThread:2026123 [sender.py:transition_state():614] send defer: 5 +2025-01-04 21:28:11,703 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: keepalive +2025-01-04 21:28:11,703 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:28:11,704 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:28:11,704 INFO HandlerThread:2026123 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-04 21:28:11,705 DEBUG SenderThread:2026123 [sender.py:send():379] send: history +2025-01-04 21:28:11,705 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:28:11,706 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:28:11,706 DEBUG SenderThread:2026123 [sender.py:send():379] send: summary +2025-01-04 21:28:11,710 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:28:11,710 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: defer +2025-01-04 21:28:11,710 INFO SenderThread:2026123 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-04 21:28:11,710 INFO SenderThread:2026123 [sender.py:transition_state():614] send defer: 6 +2025-01-04 21:28:11,710 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:28:11,710 INFO HandlerThread:2026123 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-04 21:28:11,711 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: defer +2025-01-04 21:28:11,711 INFO SenderThread:2026123 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-04 21:28:11,714 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:28:11,850 INFO SenderThread:2026123 [sender.py:transition_state():614] send defer: 7 +2025-01-04 21:28:11,850 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:28:11,850 INFO HandlerThread:2026123 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-04 21:28:11,850 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: defer +2025-01-04 21:28:11,850 INFO SenderThread:2026123 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-04 21:28:12,026 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/config.yaml +2025-01-04 21:28:12,026 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/wandb-summary.json +2025-01-04 21:28:12,182 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:28:13,515 INFO SenderThread:2026123 [sender.py:transition_state():614] send defer: 8 +2025-01-04 21:28:13,515 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:28:13,516 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:28:13,516 INFO HandlerThread:2026123 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-04 21:28:13,516 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: defer +2025-01-04 21:28:13,516 INFO SenderThread:2026123 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-04 21:28:13,516 INFO SenderThread:2026123 [job_builder.py:build():440] Attempting to build job artifact +2025-01-04 21:28:13,518 INFO SenderThread:2026123 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-04 21:28:13,520 INFO SenderThread:2026123 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-04 21:28:13,533 INFO SenderThread:2026123 [sender.py:transition_state():614] send defer: 9 +2025-01-04 21:28:13,533 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:28:13,533 INFO HandlerThread:2026123 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-04 21:28:13,533 DEBUG SenderThread:2026123 [sender.py:send():379] send: artifact +2025-01-04 21:28:13,799 INFO SenderThread:2026123 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxOTEyMTE2Ng==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTEzMDY5OQ==', 'versionIndex': 419}}} +2025-01-04 21:28:13,799 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: defer +2025-01-04 21:28:13,799 INFO SenderThread:2026123 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-04 21:28:13,799 INFO SenderThread:2026123 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-04 21:28:14,027 INFO SenderThread:2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:28:14,027 INFO SenderThread:2026123 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files +2025-01-04 21:28:14,028 INFO SenderThread:2026123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/config.yaml config.yaml +2025-01-04 21:28:14,028 INFO SenderThread:2026123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/events.out.tfevents.1736043808.leonmkim-ROG-Strix-G15CS-G15CS.2025958.0 events.out.tfevents.1736043808.leonmkim-ROG-Strix-G15CS-G15CS.2025958.0 +2025-01-04 21:28:14,028 INFO SenderThread:2026123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/wandb-summary.json wandb-summary.json +2025-01-04 21:28:14,028 INFO SenderThread:2026123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/requirements.txt requirements.txt +2025-01-04 21:28:14,028 INFO SenderThread:2026123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/wandb-metadata.json wandb-metadata.json +2025-01-04 21:28:14,028 INFO SenderThread:2026123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log output.log +2025-01-04 21:28:14,028 INFO SenderThread:2026123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/diff.patch diff.patch +2025-01-04 21:28:14,028 INFO SenderThread:2026123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-04 21:28:14,030 INFO SenderThread:2026123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval/1_eval_1_b0947b11fcc3d3b55612.mp4 media/videos/eval/1_eval_1_b0947b11fcc3d3b55612.mp4 +2025-01-04 21:28:14,030 INFO SenderThread:2026123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval/3_eval_3_c975888550477f48ba44.mp4 media/videos/eval/3_eval_3_c975888550477f48ba44.mp4 +2025-01-04 21:28:14,030 INFO SenderThread:2026123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval/4_eval_4_d357aac167e5d7bc548b.mp4 media/videos/eval/4_eval_4_d357aac167e5d7bc548b.mp4 +2025-01-04 21:28:14,030 INFO SenderThread:2026123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval/2_eval_2_8a57ace6d7baf7108296.mp4 media/videos/eval/2_eval_2_8a57ace6d7baf7108296.mp4 +2025-01-04 21:28:14,030 INFO SenderThread:2026123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval/0_eval_0_46f66c2ba3db8f12f00c.mp4 media/videos/eval/0_eval_0_46f66c2ba3db8f12f00c.mp4 +2025-01-04 21:28:14,031 INFO SenderThread:2026123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/table/eval/total_success_rate_ci_6_2ba98cbc4fea23f33f14.table.json media/table/eval/total_success_rate_ci_6_2ba98cbc4fea23f33f14.table.json +2025-01-04 21:28:14,031 INFO SenderThread:2026123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-04 21:28:14,031 INFO SenderThread:2026123 [sender.py:transition_state():614] send defer: 10 +2025-01-04 21:28:14,031 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:28:14,031 INFO HandlerThread:2026123 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-04 21:28:14,037 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: defer +2025-01-04 21:28:14,037 INFO SenderThread:2026123 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-04 21:28:14,037 INFO SenderThread:2026123 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 21:28:14,183 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:28:14,184 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:28:14,234 INFO wandb-upload_1:2026123 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/config.yaml +2025-01-04 21:28:14,238 INFO wandb-upload_2:2026123 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/wandb-summary.json +2025-01-04 21:28:14,265 INFO wandb-upload_3:2026123 [upload_job.py:push():130] Uploaded file /tmp/tmpq2mmia7ewandb/6tlriv4l-events.out.tfevents.1736043808.leonmkim-ROG-Strix-G15CS-G15CS.2025958.0 +2025-01-04 21:28:14,269 INFO wandb-upload_4:2026123 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/requirements.txt +2025-01-04 21:28:14,271 INFO wandb-upload_0:2026123 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:28:14,471 INFO Thread-11 (_thread_body):2026123 [sender.py:transition_state():614] send defer: 11 +2025-01-04 21:28:14,471 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:28:14,472 INFO HandlerThread:2026123 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-04 21:28:14,472 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: defer +2025-01-04 21:28:14,472 INFO SenderThread:2026123 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-04 21:28:14,472 INFO SenderThread:2026123 [file_pusher.py:join():175] waiting for file pusher +2025-01-04 21:28:14,473 INFO SenderThread:2026123 [sender.py:transition_state():614] send defer: 12 +2025-01-04 21:28:14,473 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:28:14,473 INFO HandlerThread:2026123 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-04 21:28:14,473 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: defer +2025-01-04 21:28:14,473 INFO SenderThread:2026123 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-04 21:28:14,473 INFO SenderThread:2026123 [file_stream.py:finish():601] file stream finish called +2025-01-04 21:28:14,670 INFO SenderThread:2026123 [file_stream.py:finish():605] file stream finish is done +2025-01-04 21:28:14,670 INFO SenderThread:2026123 [sender.py:transition_state():614] send defer: 13 +2025-01-04 21:28:14,670 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:28:14,670 INFO HandlerThread:2026123 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-04 21:28:14,670 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: defer +2025-01-04 21:28:14,670 INFO SenderThread:2026123 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-04 21:28:14,671 INFO SenderThread:2026123 [sender.py:transition_state():614] send defer: 14 +2025-01-04 21:28:14,671 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:28:14,671 INFO HandlerThread:2026123 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-04 21:28:14,671 DEBUG SenderThread:2026123 [sender.py:send():379] send: final +2025-01-04 21:28:14,671 DEBUG SenderThread:2026123 [sender.py:send():379] send: footer +2025-01-04 21:28:14,671 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: defer +2025-01-04 21:28:14,671 INFO SenderThread:2026123 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-04 21:28:14,671 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:28:14,672 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:28:14,672 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: server_info +2025-01-04 21:28:14,672 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:28:14,672 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: get_summary +2025-01-04 21:28:14,673 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: server_info +2025-01-04 21:28:14,674 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-04 21:28:14,675 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:28:14,733 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:28:14,734 INFO MainThread:2026123 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-04 21:28:14,734 INFO MainThread:2026123 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-04 21:28:14,734 INFO MainThread:2026123 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-04 21:28:14,735 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: shutdown +2025-01-04 21:28:14,735 INFO HandlerThread:2026123 [handler.py:finish():882] shutting down handler +2025-01-04 21:28:15,673 INFO WriterThread:2026123 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/run-chnx0pdg.wandb +2025-01-04 21:28:15,734 INFO SenderThread:2026123 [sender.py:finish():1615] shutting down sender +2025-01-04 21:28:15,734 INFO SenderThread:2026123 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 21:28:15,734 INFO SenderThread:2026123 [file_pusher.py:join():175] waiting for file pusher diff --git a/chnx0pdg/wandb/debug.log b/chnx0pdg/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..70ea78ac90294a3f323f5ccb9944db93f94c89ee --- /dev/null +++ b/chnx0pdg/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-04 21:23:27,657 INFO MainThread:2025958 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-04 21:23:27,657 INFO MainThread:2025958 [wandb_setup.py:_flush():76] Configure stats pid to 2025958 +2025-01-04 21:23:27,657 INFO MainThread:2025958 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-04 21:23:27,657 INFO MainThread:2025958 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/settings +2025-01-04 21:23:27,657 INFO MainThread:2025958 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-04 21:23:27,657 INFO MainThread:2025958 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-04 21:23:27,657 INFO MainThread:2025958 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-04 21:23:27,657 INFO MainThread:2025958 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-04 21:23:27,657 INFO MainThread:2025958 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/logs/debug.log +2025-01-04 21:23:27,657 INFO MainThread:2025958 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/logs/debug-internal.log +2025-01-04 21:23:27,657 INFO MainThread:2025958 [wandb_init.py:init():569] calling init triggers +2025-01-04 21:23:27,657 INFO MainThread:2025958 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1938_0', 'wandb_notes': '1938_0_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg'} +2025-01-04 21:23:27,658 INFO MainThread:2025958 [wandb_init.py:init():619] starting backend +2025-01-04 21:23:27,658 INFO MainThread:2025958 [wandb_init.py:init():623] setting up manager +2025-01-04 21:23:27,662 INFO MainThread:2025958 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-04 21:23:27,664 INFO MainThread:2025958 [wandb_init.py:init():631] backend started and connected +2025-01-04 21:23:27,675 INFO MainThread:2025958 [wandb_init.py:init():720] updated telemetry +2025-01-04 21:23:27,683 INFO MainThread:2025958 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-04 21:23:27,942 INFO MainThread:2025958 [wandb_run.py:_on_init():2435] communicating current version +2025-01-04 21:23:27,984 INFO MainThread:2025958 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-04 21:23:27,984 INFO MainThread:2025958 [wandb_init.py:init():804] starting run threads in backend +2025-01-04 21:23:28,312 INFO MainThread:2025958 [wandb_run.py:_console_start():2413] atexit reg +2025-01-04 21:23:28,313 INFO MainThread:2025958 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-04 21:23:28,313 INFO MainThread:2025958 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-04 21:23:28,313 INFO MainThread:2025958 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-04 21:23:28,316 INFO MainThread:2025958 [wandb_init.py:init():847] run started, returning control to user process +2025-01-04 21:23:28,316 INFO MainThread:2025958 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/tb, True +2025-01-04 21:23:32,405 INFO MainThread:2025958 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-04 21:28:15,735 WARNING MsgRouterThr:2025958 [router.py:message_loop():77] message_loop has been closed diff --git a/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/code/FISH/eval_robot.py b/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/config.yaml b/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..431d69aebafdac60c2f7017c0c661864a21d28eb --- /dev/null +++ b/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 1 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1938_0' +wandb_notes: + desc: null + value: 1938_0_restarted_5 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736043807 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/diff.patch b/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..c70dbd91a4993732d181d425e68ce116710765ed --- /dev/null +++ b/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/diff.patch @@ -0,0 +1,56 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 57a851f..816d960 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -138,12 +138,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' ++# wandb_run_id: '1922_1' # seed 1 ++wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1006_0' ++# wandb_run_id: '1926_1' # seed 1 ++# wandb_run_id: '1930_0' # dataset shuffle seed 1 ++ ++ + + # RGBD+mask+contact(contact+DTC)+act history +-wandb_run_id: '1745_0' ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 71a5d1c..c7eadce 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -159,8 +159,10 @@ run_id_list = [ + ] + + run_id_list = [ +- # '1006_0', +- '1745_0' ++ # '1922_1', ++ # '1922_2', ++ '1926_1', ++ '1930_0', + ] + + checkpoint_epoch = 12000 +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index 6f97e26..285afc8 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,8 +330,8 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.005 +- self.move_to_rot_tolerance = 1.0 ++ self.move_to_trans_tolerance = 0.01 ++ self.move_to_rot_tolerance = 2.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) + self.move_to_target_timeout = 5.0 diff --git a/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json b/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json new file mode 100644 index 0000000000000000000000000000000000000000..abffe612675a3f5ba7b32314013cd92e261fa3b0 --- /dev/null +++ b/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.449680686007073, 1, 0.47999376058978616, 1, 0.6638181209236573, 1, 1, 1]]} \ No newline at end of file diff --git a/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/table/eval/total_success_rate_ci_6_2ba98cbc4fea23f33f14.table.json b/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/table/eval/total_success_rate_ci_6_2ba98cbc4fea23f33f14.table.json new file mode 100644 index 0000000000000000000000000000000000000000..0688daf9c9e63e02548e7f2b22545029ebfbf063 --- /dev/null +++ b/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/table/eval/total_success_rate_ci_6_2ba98cbc4fea23f33f14.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[48, 50, 0.96, 0.8497122823492624, 0.9960396214327554, 0.8803341164424305, 0.9949651741364105, 0.8956393226189883, 0.9885692316061636, 0.9012663065584643, 0.9885647114320486]]} \ No newline at end of file diff --git a/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval/0_eval_0_46f66c2ba3db8f12f00c.mp4 b/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval/0_eval_0_46f66c2ba3db8f12f00c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..25d7a346c48cbd18adb166e216b3389c4c3f08e3 --- /dev/null +++ b/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval/0_eval_0_46f66c2ba3db8f12f00c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46f66c2ba3db8f12f00c5bc7aa19ba0e5eb6f4f2c12aa8227d2e23c96c9b96b8 +size 1195059 diff --git a/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval/1_eval_1_b0947b11fcc3d3b55612.mp4 b/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval/1_eval_1_b0947b11fcc3d3b55612.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a7e88088835eb267a9aed5ec8b4b3daa0af0eb07 --- /dev/null +++ b/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval/1_eval_1_b0947b11fcc3d3b55612.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0947b11fcc3d3b55612d61edb7fd1f3364879e96b5a1b96730b590cccc47ed5 +size 1193365 diff --git a/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval/2_eval_2_8a57ace6d7baf7108296.mp4 b/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval/2_eval_2_8a57ace6d7baf7108296.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2f670a247e39382dd7343367e65cf55b4f827f04 --- /dev/null +++ b/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval/2_eval_2_8a57ace6d7baf7108296.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a57ace6d7baf7108296c5eb6844321ff8138f5cfd494b222e13551ec8e9aa68 +size 1196574 diff --git a/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval/3_eval_3_c975888550477f48ba44.mp4 b/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval/3_eval_3_c975888550477f48ba44.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..33a9fea0d0949387d46cd21c61c0e9ab9e7be789 --- /dev/null +++ b/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval/3_eval_3_c975888550477f48ba44.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c975888550477f48ba44a40bea300cfb9a0494e1be02f664ae0080151df75294 +size 1192068 diff --git a/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval/4_eval_4_d357aac167e5d7bc548b.mp4 b/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval/4_eval_4_d357aac167e5d7bc548b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..65ff415da7e1478410c680067527b9b5e547ac1d --- /dev/null +++ b/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval/4_eval_4_d357aac167e5d7bc548b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d357aac167e5d7bc548b9c4a8147292bbc6865983c746e7848689ead6f3ad4e3 +size 1202203 diff --git a/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log b/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..364dcfe5a03004d045d0c2e38bd7f81c99c11bfc --- /dev/null +++ b/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log @@ -0,0 +1,281 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736043812.406235]: resetting environment +[INFO] [1736043812.409042]: cleared current plan +[INFO] [1736043812.409846]: moving to home +[INFO] [1736043813.413290]: reached home +[INFO] [1736043813.414554]: reset action history +[INFO] [1736043816.123459]: environment reset +Starting episode 0 +[INFO] [1736043816.125679]: resetting environment +[INFO] [1736043816.128972]: cleared current plan +[INFO] [1736043816.129984]: moving to home +[INFO] [1736043817.132607]: reached home +[INFO] [1736043817.133887]: reset action history +[INFO] [1736043819.841697]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736043831.669553]: resetting environment +[INFO] [1736043831.675106]: cleared current plan +[INFO] [1736043831.676502]: moving to home +[INFO] [1736043832.678849]: reached home +[INFO] [1736043832.680357]: reset action history +[INFO] [1736043835.388175]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/episode_rosbags/episode_0_2025-01-04-21-23-55.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2025958.log +started roslaunch server http://158.130.50.37:38059/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2026247] +process[print_text-2]: started with pid [2026248] +process[rosbag_record-3]: started with pid [2026249] +started bagging! +For topic gripper_width: timestamp difference is 113111571 for nearest: 1736043838618222073 - target: 1736043838505110502 at idx 46 +For topic gripper_width: timestamp difference is 112047790 for nearest: 1736043838818247674 - target: 1736043838706199884 at idx 48 +For topic gripper_width: timestamp difference is 110314828 for nearest: 1736043840018241149 - target: 1736043839907926321 at idx 64 +[INFO] [1736043862.397970]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736043863.124326]: Stored episode 1. +[INFO] [1736043863.124584]: Saving video... +[INFO] [1736043863.310815]: Video saved! +Starting episode 1 +[INFO] [1736043863.328581]: resetting environment +[INFO] [1736043863.341735]: cleared current plan +[INFO] [1736043863.342009]: moving to home +[INFO] [1736043866.943830]: reached home +[INFO] [1736043866.944658]: reset action history +[INFO] [1736043869.650840]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736043875.337521]: resetting environment +[INFO] [1736043875.340882]: cleared current plan +[INFO] [1736043875.341037]: moving to home +[INFO] [1736043876.342463]: reached home +[INFO] [1736043876.342862]: reset action history +[INFO] [1736043879.051624]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/episode_rosbags/episode_1_2025-01-04-21-24-39.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2025958.log +started roslaunch server http://158.130.50.37:35157/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2026358] +process[print_text-5]: started with pid [2026359] +process[rosbag_record-6]: started with pid [2026383] +started bagging! +For topic gripper_width: timestamp difference is -209003940 for nearest: 1736043898351505026 - target: 1736043898560508966 at idx 249 +For topic gripper_width: timestamp difference is -106744803 for nearest: 1736043901251549684 - target: 1736043901358294487 at idx 249 +For topic gripper_width: timestamp difference is -104790260 for nearest: 1736043903651546667 - target: 1736043903756336927 at idx 249 +[INFO] [1736043906.024609]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1736043906.788850]: Stored episode 2. +[INFO] [1736043906.789118]: Saving video... +[INFO] [1736043907.432239]: Video saved! +Starting episode 2 +[INFO] [1736043907.472025]: resetting environment +[INFO] [1736043907.490503]: cleared current plan +[INFO] [1736043907.495381]: moving to home +[WARN] [1736043907.344093]: Plan exhausted +[WARN] [1736043907.384816]: Plan exhausted +[WARN] [1736043907.430186]: Plan exhausted +[WARN] [1736043907.464108]: Plan exhausted +[INFO] [1736043911.099836]: reached home +[INFO] [1736043911.101585]: reset action history +[INFO] [1736043913.873784]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736043944.530814]: resetting environment +[INFO] [1736043944.534803]: cleared current plan +[INFO] [1736043944.535073]: moving to home +[INFO] [1736043945.535728]: reached home +[INFO] [1736043945.536191]: reset action history +[INFO] [1736043948.300011]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/episode_rosbags/episode_2_2025-01-04-21-25-48.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2025958.log +started roslaunch server http://158.130.50.37:35891/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2026498] +process[print_text-8]: started with pid [2026499] +process[rosbag_record-9]: started with pid [2026523] +started bagging! +[INFO] [1736043975.290248]: Storing episode... +[rosbag_record-9] killing on exit +[pub_text-7] killing on exit +[print_text-8] killing on exit +[WARN] [1736043976.625908]: Plan exhausted +[WARN] [1736043976.668066]: Plan exhausted +[WARN] [1736043976.704602]: Plan exhausted +[INFO] [1736043976.072062]: Stored episode 3. +[INFO] [1736043976.074307]: Saving video... +[INFO] [1736043976.680108]: Video saved! +Starting episode 3 +[INFO] [1736043976.720543]: resetting environment +[INFO] [1736043976.732253]: cleared current plan +[INFO] [1736043976.732674]: moving to home +[INFO] [1736043980.338458]: reached home +[INFO] [1736043980.344599]: reset action history +[INFO] [1736043983.122828]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736043986.469533]: resetting environment +[INFO] [1736043986.485253]: cleared current plan +[INFO] [1736043986.489412]: moving to home +[INFO] [1736043987.502755]: reached home +[INFO] [1736043987.506121]: reset action history +[INFO] [1736043990.408111]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/episode_rosbags/episode_3_2025-01-04-21-26-30.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2025958.log +started roslaunch server http://158.130.50.37:37147/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2026633] +process[print_text-11]: started with pid [2026634] +process[rosbag_record-12]: started with pid [2026658] +started bagging! +For topic gripper_width: timestamp difference is 103604520 for nearest: 1736043996618245567 - target: 1736043996514641047 at idx 90 +[INFO] [1736044017.386305]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[WARN] [1736044018.713122]: Plan exhausted +[WARN] [1736044018.755737]: Plan exhausted +[WARN] [1736044018.793000]: Plan exhausted +[WARN] [1736044018.832997]: Plan exhausted +[WARN] [1736044018.886852]: Plan exhausted +[WARN] [1736044018.923273]: Plan exhausted +[WARN] [1736044018.953211]: Plan exhausted +[INFO] [1736044018.169272]: Stored episode 4. +[INFO] [1736044018.178977]: Saving video... +[INFO] [1736044018.914921]: Video saved! +Starting episode 4 +[INFO] [1736044018.969205]: resetting environment +[INFO] [1736044018.996995]: cleared current plan +[INFO] [1736044018.997336]: moving to home +[WARN] [1736044018.992929]: Plan exhausted +[INFO] [1736044022.603380]: reached home +[INFO] [1736044022.616086]: reset action history +[INFO] [1736044025.457324]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736044029.713432]: resetting environment +[INFO] [1736044029.746864]: cleared current plan +[INFO] [1736044029.749896]: moving to home +[INFO] [1736044030.756047]: reached home +[INFO] [1736044030.771738]: reset action history +[INFO] [1736044033.551566]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/episode_rosbags/episode_4_2025-01-04-21-27-13.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2025958.log +started roslaunch server http://158.130.50.37:46461/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2026768] +process[print_text-14]: started with pid [2026769] +process[rosbag_record-15]: started with pid [2026793] +started bagging! +[INFO] [1736044060.544993]: Storing episode... +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit[print_text-14] killing on exit +[INFO] [1736044061.347073]: Stored episode 5. +[INFO] [1736044061.350570]: Saving video... +[INFO] [1736044061.780777]: Video saved! +[INFO] [1736044061.849230]: resetting environment +[INFO] [1736044061.860643]: cleared current plan +[INFO] [1736044061.863082]: moving to home +[INFO] [1736044065.470258]: reached home +[INFO] [1736044065.480904]: reset action history +[INFO] [1736044068.343266]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/requirements.txt b/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/wandb-metadata.json b/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..dce2c15cc9199ec3da50ba6ffc54832875571481 --- /dev/null +++ b/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-05T02:23:28.006710", + "startedAt": "2025-01-05T02:23:27.651297", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "02cf07d901f3d8187a302cd51c9faa80321417ee" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 4125.005625, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.998, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.951, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 669.7662467956543 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/wandb-summary.json b/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..988202cc200eedc072fa44fabac2c884b1277fd8 --- /dev/null +++ b/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "46f66c2ba3db8f12f00c5bc7aa19ba0e5eb6f4f2c12aa8227d2e23c96c9b96b8", "size": 1195059, "path": "media/videos/eval/0_eval_0_46f66c2ba3db8f12f00c.mp4"}, "global_step": 665, "_timestamp": 1736044078.4850378, "_runtime": 270.82086181640625, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "b0947b11fcc3d3b55612d61edb7fd1f3364879e96b5a1b96730b590cccc47ed5", "size": 1193365, "path": "media/videos/eval/1_eval_1_b0947b11fcc3d3b55612.mp4"}, "eval/num_success": 5.0, "episode": 50.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.4496806859970093, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.4799937605857849, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6638181209564209, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "8a57ace6d7baf7108296c5eb6844321ff8138f5cfd494b222e13551ec8e9aa68", "size": 1196574, "path": "media/videos/eval/2_eval_2_8a57ace6d7baf7108296.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "c975888550477f48ba44a40bea300cfb9a0494e1be02f664ae0080151df75294", "size": 1192068, "path": "media/videos/eval/3_eval_3_c975888550477f48ba44.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "d357aac167e5d7bc548b9c4a8147292bbc6865983c746e7848689ead6f3ad4e3", "size": 1202203, "path": "media/videos/eval/4_eval_4_d357aac167e5d7bc548b.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "1b420c32327aa0ba7f7504ab8676a845cd69d26134af4445f97766065563f5e1", "size": 313, "artifact_path": "wandb-client-artifact://kk0phh0y2irelv63j2k4qkuxrlowvongqcvni5hocsqxoczwwq2udhkt0m8a3av1xwge8z41quj03jziuxhnoa118otu6ersdfiol1uztd1ndtf4xwfi74vl3poa7u6x/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://k367oat03l2wqjim798cgd0df1repqeov3kcm5x6zkpydeqs3ocap005drpd8d2kxjx3jdg2ssbg5e07ax491p2p3fqxurttebr6296ox13u163hp83zkkbc0pujqb15:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 48, "total_num_episodes": 50, "total_success_rate": 0.96, "total_success_rate_lb_0.01": 0.8497122823492624, "total_success_rate_ub_0.01": 0.9960396214327554, "total_success_rate_lb_0.025": 0.8803341164424305, "total_success_rate_ub_0.025": 0.9949651741364105, "total_success_rate_lb_0.05": 0.8956393226189883, "total_success_rate_ub_0.05": 0.9885692316061636, "total_success_rate_lb_0.1": 0.9012663065584643, "total_success_rate_ub_0.1": 0.9885647114320486, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "2ba98cbc4fea23f33f1441c6998e15f8238ad0040a1049ed00e1b5d253c65630", "size": 506, "artifact_path": "wandb-client-artifact://9mcs4szbxz5c0jmtgknzko0362nz259ssrs3w2cb0zr024xcck69xs75w5yv986kmu335byl6rspkaxf75r9weljsk2x26791mylvgemeaxn279lyt6dron7zlnz81e5/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://la2vnk4sptv7cvfn1yxw2jvg6k4bymqddeumhz8mvpactj7wmbuojky1yjwvojuoaxkm5ndfqxyzv1df8scuv011ydjeztxccj72mihfdcpn3xtyfz6oxi8os5pc4xqw:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_2ba98cbc4fea23f33f14.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 277}, "eval/total_success_rate": 0.9599999785423279, "eval/total_success_rate_lb_0.01": 0.8497122526168823, "eval/total_success_rate_ub_0.01": 0.996039628982544, "eval/total_success_rate_lb_0.025": 0.8803341388702393, "eval/total_success_rate_ub_0.025": 0.9949651956558228, "eval/total_success_rate_lb_0.05": 0.8956393003463745, "eval/total_success_rate_ub_0.05": 0.9885692596435547, "eval/total_success_rate_lb_0.1": 0.9012662768363953, "eval/total_success_rate_ub_0.1": 0.9885647296905518} \ No newline at end of file diff --git a/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/logs/debug-internal.log b/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..b74024d555f04fd296f4d314b8b2e8db17c7a3b0 --- /dev/null +++ b/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/logs/debug-internal.log @@ -0,0 +1,784 @@ +2025-01-04 21:23:27,663 INFO StreamThr :2026123 [internal.py:wandb_internal():85] W&B internal server running at pid: 2026123, started at: 2025-01-04 21:23:27.662939 +2025-01-04 21:23:27,664 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status +2025-01-04 21:23:27,675 INFO WriterThread:2026123 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/run-chnx0pdg.wandb +2025-01-04 21:23:27,676 DEBUG SenderThread:2026123 [sender.py:send():379] send: header +2025-01-04 21:23:27,685 DEBUG SenderThread:2026123 [sender.py:send():379] send: run +2025-01-04 21:23:27,927 INFO SenderThread:2026123 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files +2025-01-04 21:23:27,927 INFO SenderThread:2026123 [sender.py:_start_run_threads():1188] run started: chnx0pdg with start time 1736043807.664176 +2025-01-04 21:23:27,942 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: check_version +2025-01-04 21:23:27,943 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: check_version +2025-01-04 21:23:27,988 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: run_start +2025-01-04 21:23:27,993 DEBUG HandlerThread:2026123 [system_info.py:__init__():26] System info init +2025-01-04 21:23:27,993 DEBUG HandlerThread:2026123 [system_info.py:__init__():41] System info init done +2025-01-04 21:23:27,993 INFO HandlerThread:2026123 [system_monitor.py:start():194] Starting system monitor +2025-01-04 21:23:27,993 INFO SystemMonitor:2026123 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-04 21:23:27,994 INFO SystemMonitor:2026123 [interfaces.py:start():188] Started cpu monitoring +2025-01-04 21:23:27,994 INFO SystemMonitor:2026123 [interfaces.py:start():188] Started disk monitoring +2025-01-04 21:23:27,995 INFO HandlerThread:2026123 [system_monitor.py:probe():214] Collecting system info +2025-01-04 21:23:27,995 INFO SystemMonitor:2026123 [interfaces.py:start():188] Started gpu monitoring +2025-01-04 21:23:27,996 INFO SystemMonitor:2026123 [interfaces.py:start():188] Started memory monitoring +2025-01-04 21:23:27,996 INFO SystemMonitor:2026123 [interfaces.py:start():188] Started network monitoring +2025-01-04 21:23:28,006 DEBUG HandlerThread:2026123 [system_info.py:probe():152] Probing system +2025-01-04 21:23:28,008 DEBUG HandlerThread:2026123 [system_info.py:_probe_git():137] Probing git +2025-01-04 21:23:28,012 DEBUG HandlerThread:2026123 [system_info.py:_probe_git():145] Probing git done +2025-01-04 21:23:28,012 DEBUG HandlerThread:2026123 [system_info.py:probe():200] Probing system done +2025-01-04 21:23:28,012 DEBUG HandlerThread:2026123 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-05T02:23:28.006710', 'startedAt': '2025-01-05T02:23:27.651297', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '02cf07d901f3d8187a302cd51c9faa80321417ee'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4125.005625, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.998, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.951, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 669.7662467956543}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-04 21:23:28,012 INFO HandlerThread:2026123 [system_monitor.py:probe():224] Finished collecting system info +2025-01-04 21:23:28,012 INFO HandlerThread:2026123 [system_monitor.py:probe():227] Publishing system info +2025-01-04 21:23:28,012 DEBUG HandlerThread:2026123 [system_info.py:_save_code():44] Saving code +2025-01-04 21:23:28,014 DEBUG HandlerThread:2026123 [system_info.py:_save_code():67] Saving code done +2025-01-04 21:23:28,014 DEBUG HandlerThread:2026123 [system_info.py:_save_patches():84] Saving git patches +2025-01-04 21:23:28,078 DEBUG HandlerThread:2026123 [system_info.py:_save_patches():126] Saving git patches done +2025-01-04 21:23:28,079 INFO HandlerThread:2026123 [system_monitor.py:probe():229] Finished publishing system info +2025-01-04 21:23:28,082 DEBUG SenderThread:2026123 [sender.py:send():379] send: files +2025-01-04 21:23:28,082 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-04 21:23:28,082 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-04 21:23:28,083 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-04 21:23:28,312 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: python_packages +2025-01-04 21:23:28,312 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:23:28,313 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: python_packages +2025-01-04 21:23:28,313 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:23:28,315 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:28,317 INFO HandlerThread:2026123 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/tb" + save: true +} +control { + relay_id: "59aad6f3f1b84a2a823baf79aed48f1c" +} +_info { + stream_id: "chnx0pdg" +} + +2025-01-04 21:23:28,317 DEBUG HandlerThread:2026123 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-04 21:23:28,324 INFO wandb-upload_0:2026123 [upload_job.py:push():130] Uploaded file /tmp/tmpq2mmia7ewandb/q93gcekk-wandb-metadata.json +2025-01-04 21:23:28,362 INFO wandb-upload_2:2026123 [upload_job.py:push():130] Uploaded file /tmp/tmpq2mmia7ewandb/9npmef1b-diff.patch +2025-01-04 21:23:28,380 DEBUG SenderThread:2026123 [sender.py:send():379] send: telemetry +2025-01-04 21:23:28,381 DEBUG SenderThread:2026123 [sender.py:send():379] send: tbrecord +2025-01-04 21:23:28,381 DEBUG SenderThread:2026123 [sender.py:send():379] send: files +2025-01-04 21:23:28,381 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file events.out.tfevents.1736043808.leonmkim-ROG-Strix-G15CS-G15CS.2025958.0 with policy live +2025-01-04 21:23:28,394 INFO wandb-upload_1:2026123 [upload_job.py:push():130] Uploaded file /tmp/tmpq2mmia7ewandb/1qfyl7ip-code/FISH/eval_robot.py +2025-01-04 21:23:28,580 INFO wandb-upload_0:2026123 [upload_job.py:push():130] Uploaded file /tmp/tmpq2mmia7ewandb/27s3y4w3-events.out.tfevents.1736043808.leonmkim-ROG-Strix-G15CS-G15CS.2025958.0 +2025-01-04 21:23:28,929 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:23:28,929 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/code/FISH/eval_robot.py +2025-01-04 21:23:28,929 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/events.out.tfevents.1736043808.leonmkim-ROG-Strix-G15CS-G15CS.2025958.0 +2025-01-04 21:23:28,929 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/requirements.txt +2025-01-04 21:23:28,929 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/diff.patch +2025-01-04 21:23:28,929 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/wandb-metadata.json +2025-01-04 21:23:28,929 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/code +2025-01-04 21:23:28,929 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/code/FISH +2025-01-04 21:23:29,313 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:30,313 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:30,928 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:23:31,313 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:32,314 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:32,406 DEBUG SenderThread:2026123 [sender.py:send():379] send: config +2025-01-04 21:23:33,313 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:23:33,314 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:34,314 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:34,929 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:23:35,314 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:36,315 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:37,314 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:38,313 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:23:38,314 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:38,930 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:23:39,315 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:40,315 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:40,931 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:23:41,315 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:42,315 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:43,312 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:23:43,313 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:23:43,353 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:43,404 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:23:44,315 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:45,315 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:46,315 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:47,316 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:48,316 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:49,314 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:23:49,316 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:50,316 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:51,316 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:52,316 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:52,934 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:23:53,317 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:54,317 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:54,935 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:23:55,314 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:23:55,317 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:56,317 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:56,935 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:23:57,317 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:58,313 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:23:58,313 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:23:58,353 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:59,317 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:00,317 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:00,936 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:24:01,069 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:24:01,317 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:01,937 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/config.yaml +2025-01-04 21:24:02,317 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:02,937 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:24:03,318 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:04,318 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:05,319 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:06,314 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:24:06,320 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:07,320 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:08,320 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:09,320 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:10,320 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:11,314 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:24:11,320 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:12,321 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:13,313 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:24:13,313 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:24:13,364 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:14,321 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:15,321 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:16,321 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:16,400 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:24:17,321 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:18,321 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:19,321 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:20,322 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:21,322 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:22,314 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:24:22,322 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:23,323 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:23,327 DEBUG SenderThread:2026123 [sender.py:send():379] send: files +2025-01-04 21:24:23,327 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_46f66c2ba3db8f12f00c.mp4 with policy now +2025-01-04 21:24:23,373 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:24:23,374 DEBUG SenderThread:2026123 [sender.py:send():379] send: history +2025-01-04 21:24:23,374 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:24:23,375 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:24:23,589 INFO wandb-upload_2:2026123 [upload_job.py:push():130] Uploaded file /tmp/tmpq2mmia7ewandb/i0snrcat-media/videos/eval/0_eval_0_46f66c2ba3db8f12f00c.mp4 +2025-01-04 21:24:23,943 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/wandb-summary.json +2025-01-04 21:24:23,943 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval/0_eval_0_46f66c2ba3db8f12f00c.mp4 +2025-01-04 21:24:23,943 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos +2025-01-04 21:24:23,943 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media +2025-01-04 21:24:23,943 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval +2025-01-04 21:24:24,206 INFO wandb-upload_0:2026123 [upload_job.py:push():130] Uploaded file /tmp/tmpq2mmia7ewandb/4gjf1xvr-media/videos/eval/0_eval_0_46f66c2ba3db8f12f00c.mp4 +2025-01-04 21:24:24,322 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:24,943 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:24:25,323 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:26,323 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:27,323 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:27,989 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:24:27,996 DEBUG SystemMonitor:2026123 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-04 21:24:27,997 DEBUG SenderThread:2026123 [sender.py:send():379] send: stats +2025-01-04 21:24:28,313 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:24:28,313 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:24:28,361 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:28,944 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:24:29,324 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:30,323 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:30,945 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:24:31,323 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:32,323 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:33,314 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:24:33,323 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:34,324 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:35,324 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:35,947 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/events.out.tfevents.1736043808.leonmkim-ROG-Strix-G15CS-G15CS.2025958.0 +2025-01-04 21:24:36,180 INFO wandb-upload_2:2026123 [upload_job.py:push():130] Uploaded file /tmp/tmpq2mmia7ewandb/9teuzw8i-events.out.tfevents.1736043808.leonmkim-ROG-Strix-G15CS-G15CS.2025958.0 +2025-01-04 21:24:36,324 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:36,947 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:24:37,324 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:38,314 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:24:38,324 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:38,948 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:24:39,324 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:40,324 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:40,948 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:24:41,324 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:42,324 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:43,313 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:24:43,313 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:24:43,357 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:43,471 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:24:44,327 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:45,327 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:46,327 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:47,328 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:48,328 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:49,315 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:24:49,328 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:50,330 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:51,330 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:52,330 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:53,330 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:54,315 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:24:54,331 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:55,331 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:56,331 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:57,331 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:57,997 DEBUG SenderThread:2026123 [sender.py:send():379] send: stats +2025-01-04 21:24:58,313 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:24:58,313 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:24:58,361 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:59,331 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:24:59,701 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:25:00,331 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:00,955 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:25:01,331 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:02,331 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:02,955 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:25:03,331 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:04,332 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:04,885 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:25:04,956 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:25:05,332 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:06,332 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:07,334 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:07,464 DEBUG SenderThread:2026123 [sender.py:send():379] send: files +2025-01-04 21:25:07,464 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_b0947b11fcc3d3b55612.mp4 with policy now +2025-01-04 21:25:07,467 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:25:07,467 DEBUG SenderThread:2026123 [sender.py:send():379] send: history +2025-01-04 21:25:07,468 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:25:07,468 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:25:07,725 INFO wandb-upload_0:2026123 [upload_job.py:push():130] Uploaded file /tmp/tmpq2mmia7ewandb/yyibyam3-media/videos/eval/1_eval_1_b0947b11fcc3d3b55612.mp4 +2025-01-04 21:25:07,957 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/wandb-summary.json +2025-01-04 21:25:07,957 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval/1_eval_1_b0947b11fcc3d3b55612.mp4 +2025-01-04 21:25:07,957 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval +2025-01-04 21:25:08,237 INFO wandb-upload_2:2026123 [upload_job.py:push():130] Uploaded file /tmp/tmpq2mmia7ewandb/sucky324-media/videos/eval/1_eval_1_b0947b11fcc3d3b55612.mp4 +2025-01-04 21:25:08,341 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:08,957 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:25:09,337 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:09,958 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:25:10,331 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:25:10,339 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:11,341 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:12,340 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:12,959 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:25:13,325 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:25:13,325 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:25:13,365 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:14,338 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:14,960 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:25:15,338 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:15,917 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:25:16,339 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:17,339 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:18,339 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:19,339 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:20,341 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:21,316 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:25:21,339 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:22,340 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:23,340 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:24,340 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:25,340 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:26,316 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:25:26,340 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:27,340 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:27,998 DEBUG SenderThread:2026123 [sender.py:send():379] send: stats +2025-01-04 21:25:28,314 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:25:28,314 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:25:28,357 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:29,341 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:30,344 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:31,341 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:31,456 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:25:32,341 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:33,341 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:34,341 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:35,342 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:36,342 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:37,317 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:25:37,345 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:38,345 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:39,346 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:40,344 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:41,345 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:42,317 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:25:42,345 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:43,314 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:25:43,314 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:25:43,361 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:44,345 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:44,760 DEBUG SenderThread:2026123 [sender.py:send():379] send: history +2025-01-04 21:25:44,760 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:25:44,760 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:25:44,969 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/events.out.tfevents.1736043808.leonmkim-ROG-Strix-G15CS-G15CS.2025958.0 +2025-01-04 21:25:44,970 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/wandb-summary.json +2025-01-04 21:25:45,256 INFO wandb-upload_0:2026123 [upload_job.py:push():130] Uploaded file /tmp/tmpq2mmia7ewandb/9905gted-events.out.tfevents.1736043808.leonmkim-ROG-Strix-G15CS-G15CS.2025958.0 +2025-01-04 21:25:45,346 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:46,345 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:46,970 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:25:47,345 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:47,577 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:25:48,345 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:49,346 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:50,346 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:50,971 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:25:51,346 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:52,346 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:53,317 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:25:53,346 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:54,346 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:55,346 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:56,346 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:57,346 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:57,998 DEBUG SenderThread:2026123 [sender.py:send():379] send: stats +2025-01-04 21:25:58,314 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:25:58,314 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:25:58,361 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:25:58,473 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:25:59,346 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:00,347 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:01,347 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:02,347 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:03,347 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:04,317 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:26:04,347 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:05,347 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:06,347 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:07,347 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:08,347 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:09,318 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:26:09,347 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:10,347 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:11,348 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:12,350 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:13,314 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:26:13,314 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:26:13,357 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:14,350 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:14,477 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:26:15,352 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:16,353 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:16,711 DEBUG SenderThread:2026123 [sender.py:send():379] send: files +2025-01-04 21:26:16,711 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_8a57ace6d7baf7108296.mp4 with policy now +2025-01-04 21:26:16,717 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:26:16,717 DEBUG SenderThread:2026123 [sender.py:send():379] send: history +2025-01-04 21:26:16,717 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:26:16,717 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:26:16,980 INFO wandb-upload_2:2026123 [upload_job.py:push():130] Uploaded file /tmp/tmpq2mmia7ewandb/ez8zh6vy-media/videos/eval/2_eval_2_8a57ace6d7baf7108296.mp4 +2025-01-04 21:26:16,981 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:26:16,981 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/wandb-summary.json +2025-01-04 21:26:16,981 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval/2_eval_2_8a57ace6d7baf7108296.mp4 +2025-01-04 21:26:16,981 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval +2025-01-04 21:26:17,281 INFO wandb-upload_0:2026123 [upload_job.py:push():130] Uploaded file /tmp/tmpq2mmia7ewandb/wwneybbg-media/videos/eval/2_eval_2_8a57ace6d7baf7108296.mp4 +2025-01-04 21:26:17,355 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:17,981 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:26:18,355 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:18,982 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:26:19,365 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:20,319 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:26:20,381 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:21,369 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:22,361 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:22,983 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:26:23,358 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:24,361 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:24,984 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:26:25,368 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:26,319 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:26:26,360 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:26,590 DEBUG SenderThread:2026123 [sender.py:send():379] send: history +2025-01-04 21:26:26,590 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:26:26,590 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:26:26,985 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/events.out.tfevents.1736043808.leonmkim-ROG-Strix-G15CS-G15CS.2025958.0 +2025-01-04 21:26:26,985 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/wandb-summary.json +2025-01-04 21:26:27,372 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:27,999 DEBUG SenderThread:2026123 [sender.py:send():379] send: stats +2025-01-04 21:26:28,325 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:26:28,325 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:26:28,365 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:28,986 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:26:29,370 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:30,397 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:31,394 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:31,672 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:26:32,393 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:32,987 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:26:33,394 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:34,394 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:35,394 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:36,394 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:37,394 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:37,663 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:26:38,395 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:38,989 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:26:39,395 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:40,397 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:41,397 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:42,397 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:43,317 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:26:43,317 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:26:43,397 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:43,454 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:26:44,397 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:45,397 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:46,397 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:47,397 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:48,397 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:49,319 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:26:49,398 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:50,398 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:51,398 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:52,398 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:53,398 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:54,398 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:55,319 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:26:55,398 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:56,398 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:57,433 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:58,000 DEBUG SenderThread:2026123 [sender.py:send():379] send: stats +2025-01-04 21:26:58,326 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:26:58,326 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:26:58,400 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:58,961 DEBUG SenderThread:2026123 [sender.py:send():379] send: files +2025-01-04 21:26:58,961 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_c975888550477f48ba44.mp4 with policy now +2025-01-04 21:26:58,965 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:26:58,966 DEBUG SenderThread:2026123 [sender.py:send():379] send: history +2025-01-04 21:26:58,966 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:26:58,966 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:26:58,998 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:26:58,998 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/wandb-summary.json +2025-01-04 21:26:58,998 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval/3_eval_3_c975888550477f48ba44.mp4 +2025-01-04 21:26:58,999 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval +2025-01-04 21:26:59,222 INFO wandb-upload_2:2026123 [upload_job.py:push():130] Uploaded file /tmp/tmpq2mmia7ewandb/vahuxfyw-media/videos/eval/3_eval_3_c975888550477f48ba44.mp4 +2025-01-04 21:26:59,399 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:26:59,494 INFO wandb-upload_0:2026123 [upload_job.py:push():130] Uploaded file /tmp/tmpq2mmia7ewandb/856h6i6a-media/videos/eval/3_eval_3_c975888550477f48ba44.mp4 +2025-01-04 21:26:59,998 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:27:00,425 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:00,998 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:27:00,999 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:27:01,406 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:01,999 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:27:02,407 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:03,409 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:04,407 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:05,000 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:27:05,410 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:06,410 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:06,509 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:27:07,001 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:27:07,411 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:08,412 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:09,410 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:10,003 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/events.out.tfevents.1736043808.leonmkim-ROG-Strix-G15CS-G15CS.2025958.0 +2025-01-04 21:27:10,241 INFO wandb-upload_2:2026123 [upload_job.py:push():130] Uploaded file /tmp/tmpq2mmia7ewandb/kehe1lgk-events.out.tfevents.1736043808.leonmkim-ROG-Strix-G15CS-G15CS.2025958.0 +2025-01-04 21:27:10,412 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:10,641 DEBUG SenderThread:2026123 [sender.py:send():379] send: history +2025-01-04 21:27:10,641 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:27:10,641 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:27:11,003 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:27:11,004 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/wandb-summary.json +2025-01-04 21:27:11,428 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:11,805 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:27:12,431 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:13,004 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:27:13,334 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:27:13,334 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:27:13,416 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:14,415 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:15,005 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:27:15,415 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:16,415 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:16,831 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:27:17,415 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:18,415 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:19,415 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:20,415 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:21,416 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:22,330 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:27:22,416 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:23,416 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:24,416 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:25,416 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:26,416 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:27,330 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:27:27,416 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:28,000 DEBUG SenderThread:2026123 [sender.py:send():379] send: stats +2025-01-04 21:27:28,320 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:27:28,320 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:27:28,417 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:29,416 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:30,417 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:31,417 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:32,418 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:32,419 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:27:33,418 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:34,417 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:35,417 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:36,418 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:37,418 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:38,330 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:27:38,418 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:39,418 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:40,418 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:41,421 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:41,847 DEBUG SenderThread:2026123 [sender.py:send():379] send: files +2025-01-04 21:27:41,847 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_d357aac167e5d7bc548b.mp4 with policy now +2025-01-04 21:27:41,850 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:27:41,850 DEBUG SenderThread:2026123 [sender.py:send():379] send: history +2025-01-04 21:27:41,850 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:27:41,850 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:27:42,015 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/wandb-summary.json +2025-01-04 21:27:42,015 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval/4_eval_4_d357aac167e5d7bc548b.mp4 +2025-01-04 21:27:42,015 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval +2025-01-04 21:27:42,105 INFO wandb-upload_0:2026123 [upload_job.py:push():130] Uploaded file /tmp/tmpq2mmia7ewandb/qycayt6k-media/videos/eval/4_eval_4_d357aac167e5d7bc548b.mp4 +2025-01-04 21:27:42,369 INFO wandb-upload_2:2026123 [upload_job.py:push():130] Uploaded file /tmp/tmpq2mmia7ewandb/yygn0fri-media/videos/eval/4_eval_4_d357aac167e5d7bc548b.mp4 +2025-01-04 21:27:42,423 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:43,015 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:27:43,325 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:27:43,325 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:27:43,413 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:27:43,425 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:44,434 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:45,432 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:46,437 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:47,017 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:27:47,438 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:48,443 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:48,444 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:27:49,446 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:50,448 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:51,018 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:27:51,451 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:52,457 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:53,458 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:54,342 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:27:54,448 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:55,460 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:56,457 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:57,469 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:57,696 DEBUG SenderThread:2026123 [sender.py:send():379] send: history +2025-01-04 21:27:57,696 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:27:57,697 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:27:57,981 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 21:27:57,981 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 21:27:58,020 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/events.out.tfevents.1736043808.leonmkim-ROG-Strix-G15CS-G15CS.2025958.0 +2025-01-04 21:27:58,020 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/wandb-summary.json +2025-01-04 21:27:58,021 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-04 21:27:58,021 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media +2025-01-04 21:27:58,021 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/table +2025-01-04 21:27:58,021 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/table/eval +2025-01-04 21:27:58,134 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:27:58,214 INFO wandb-upload_0:2026123 [upload_job.py:push():130] Uploaded file /tmp/tmpq2mmia7ewandb/noyckbw3-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-04 21:27:58,321 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:27:58,452 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:58,466 INFO wandb-upload_2:2026123 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpltq5jpvh +2025-01-04 21:27:58,581 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 21:27:58,591 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:27:58,935 INFO SenderThread:2026123 [sender.py:send_request_log_artifact():1525] logged artifact run-chnx0pdg-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE1MTY5OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1NTY4NQ==', 'latestArtifact': None}} +2025-01-04 21:27:58,935 DEBUG SenderThread:2026123 [sender.py:send():379] send: files +2025-01-04 21:27:58,935 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-04 21:27:58,935 DEBUG SenderThread:2026123 [sender.py:send():379] send: stats +2025-01-04 21:27:58,935 DEBUG SenderThread:2026123 [sender.py:send():379] send: history +2025-01-04 21:27:58,935 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:27:58,936 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:27:58,936 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:27:59,021 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/events.out.tfevents.1736043808.leonmkim-ROG-Strix-G15CS-G15CS.2025958.0 +2025-01-04 21:27:59,021 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:27:59,021 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/wandb-summary.json +2025-01-04 21:27:59,021 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/table/eval/total_success_rate_ci_6_2ba98cbc4fea23f33f14.table.json +2025-01-04 21:27:59,021 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/table/eval +2025-01-04 21:27:59,096 DEBUG SenderThread:2026123 [sender.py:send():379] send: telemetry +2025-01-04 21:27:59,096 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:27:59,096 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:27:59,096 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:27:59,097 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:27:59,097 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:27:59,097 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:27:59,097 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:27:59,098 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:27:59,098 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:27:59,098 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:27:59,098 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:27:59,099 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:27:59,099 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:27:59,099 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:27:59,099 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:27:59,099 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:27:59,100 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:27:59,100 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:27:59,100 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:27:59,101 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:27:59,101 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:27:59,101 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:27:59,101 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 21:27:59,176 INFO wandb-upload_0:2026123 [upload_job.py:push():130] Uploaded file /tmp/tmpq2mmia7ewandb/4hhuic55-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-04 21:27:59,235 INFO wandb-upload_2:2026123 [upload_job.py:push():130] Uploaded file /tmp/tmpq2mmia7ewandb/s4ncw0ro-media/table/eval/total_success_rate_ci_6_2ba98cbc4fea23f33f14.table.json +2025-01-04 21:27:59,452 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:27:59,476 INFO wandb-upload_1:2026123 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpa6t6pi25 +2025-01-04 21:27:59,847 INFO SenderThread:2026123 [sender.py:send_request_log_artifact():1525] logged artifact run-chnx0pdg-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE1MTcxMw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1NTY4Nw==', 'latestArtifact': None}} +2025-01-04 21:27:59,847 DEBUG SenderThread:2026123 [sender.py:send():379] send: files +2025-01-04 21:27:59,847 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_2ba98cbc4fea23f33f14.table.json with policy now +2025-01-04 21:27:59,847 DEBUG SenderThread:2026123 [sender.py:send():379] send: history +2025-01-04 21:27:59,848 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:27:59,848 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:27:59,849 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:28:00,021 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/wandb-summary.json +2025-01-04 21:28:00,087 INFO wandb-upload_2:2026123 [upload_job.py:push():130] Uploaded file /tmp/tmpq2mmia7ewandb/6c3bxxeq-media/table/eval/total_success_rate_ci_6_2ba98cbc4fea23f33f14.table.json +2025-01-04 21:28:00,453 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:28:01,022 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:28:01,453 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:28:02,453 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:28:03,453 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:28:04,453 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:28:05,138 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:28:05,181 DEBUG SenderThread:2026123 [sender.py:send():379] send: exit +2025-01-04 21:28:05,181 INFO SenderThread:2026123 [sender.py:send_exit():586] handling exit code: 0 +2025-01-04 21:28:05,182 INFO SenderThread:2026123 [sender.py:send_exit():588] handling runtime: 277 +2025-01-04 21:28:05,182 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:28:05,182 INFO SenderThread:2026123 [sender.py:send_exit():594] send defer +2025-01-04 21:28:05,182 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:28:05,182 INFO HandlerThread:2026123 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-04 21:28:05,182 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: defer +2025-01-04 21:28:05,182 INFO SenderThread:2026123 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-04 21:28:05,182 INFO SenderThread:2026123 [sender.py:transition_state():614] send defer: 1 +2025-01-04 21:28:05,183 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:28:05,183 INFO HandlerThread:2026123 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-04 21:28:05,183 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: defer +2025-01-04 21:28:05,183 INFO SenderThread:2026123 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-04 21:28:05,183 INFO SenderThread:2026123 [sender.py:transition_state():614] send defer: 2 +2025-01-04 21:28:05,183 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:28:05,183 INFO HandlerThread:2026123 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-04 21:28:05,183 INFO HandlerThread:2026123 [system_monitor.py:finish():203] Stopping system monitor +2025-01-04 21:28:05,183 DEBUG SystemMonitor:2026123 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-04 21:28:05,183 DEBUG SystemMonitor:2026123 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-04 21:28:05,184 INFO HandlerThread:2026123 [interfaces.py:finish():200] Joined cpu monitor +2025-01-04 21:28:05,184 INFO HandlerThread:2026123 [interfaces.py:finish():200] Joined disk monitor +2025-01-04 21:28:05,237 INFO HandlerThread:2026123 [interfaces.py:finish():200] Joined gpu monitor +2025-01-04 21:28:05,237 INFO HandlerThread:2026123 [interfaces.py:finish():200] Joined memory monitor +2025-01-04 21:28:05,238 INFO HandlerThread:2026123 [interfaces.py:finish():200] Joined network monitor +2025-01-04 21:28:05,238 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: defer +2025-01-04 21:28:05,238 INFO SenderThread:2026123 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-04 21:28:05,238 INFO SenderThread:2026123 [sender.py:transition_state():614] send defer: 3 +2025-01-04 21:28:05,238 DEBUG SenderThread:2026123 [sender.py:send():379] send: stats +2025-01-04 21:28:05,238 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:28:05,238 INFO HandlerThread:2026123 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-04 21:28:05,239 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: defer +2025-01-04 21:28:05,239 INFO SenderThread:2026123 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-04 21:28:05,239 INFO SenderThread:2026123 [sender.py:transition_state():614] send defer: 4 +2025-01-04 21:28:05,239 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:28:05,239 INFO HandlerThread:2026123 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-04 21:28:06,024 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/wandb-summary.json +2025-01-04 21:28:07,024 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:28:11,702 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:28:11,703 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: defer +2025-01-04 21:28:11,703 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:28:11,703 INFO SenderThread:2026123 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-04 21:28:11,703 INFO SenderThread:2026123 [sender.py:transition_state():614] send defer: 5 +2025-01-04 21:28:11,703 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: keepalive +2025-01-04 21:28:11,703 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:28:11,704 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:28:11,704 INFO HandlerThread:2026123 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-04 21:28:11,705 DEBUG SenderThread:2026123 [sender.py:send():379] send: history +2025-01-04 21:28:11,705 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:28:11,706 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:28:11,706 DEBUG SenderThread:2026123 [sender.py:send():379] send: summary +2025-01-04 21:28:11,710 INFO SenderThread:2026123 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:28:11,710 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: defer +2025-01-04 21:28:11,710 INFO SenderThread:2026123 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-04 21:28:11,710 INFO SenderThread:2026123 [sender.py:transition_state():614] send defer: 6 +2025-01-04 21:28:11,710 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:28:11,710 INFO HandlerThread:2026123 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-04 21:28:11,711 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: defer +2025-01-04 21:28:11,711 INFO SenderThread:2026123 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-04 21:28:11,714 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:28:11,850 INFO SenderThread:2026123 [sender.py:transition_state():614] send defer: 7 +2025-01-04 21:28:11,850 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:28:11,850 INFO HandlerThread:2026123 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-04 21:28:11,850 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: defer +2025-01-04 21:28:11,850 INFO SenderThread:2026123 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-04 21:28:12,026 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/config.yaml +2025-01-04 21:28:12,026 INFO Thread-12 :2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/wandb-summary.json +2025-01-04 21:28:12,182 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:28:13,515 INFO SenderThread:2026123 [sender.py:transition_state():614] send defer: 8 +2025-01-04 21:28:13,515 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:28:13,516 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:28:13,516 INFO HandlerThread:2026123 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-04 21:28:13,516 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: defer +2025-01-04 21:28:13,516 INFO SenderThread:2026123 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-04 21:28:13,516 INFO SenderThread:2026123 [job_builder.py:build():440] Attempting to build job artifact +2025-01-04 21:28:13,518 INFO SenderThread:2026123 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-04 21:28:13,520 INFO SenderThread:2026123 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-04 21:28:13,533 INFO SenderThread:2026123 [sender.py:transition_state():614] send defer: 9 +2025-01-04 21:28:13,533 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:28:13,533 INFO HandlerThread:2026123 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-04 21:28:13,533 DEBUG SenderThread:2026123 [sender.py:send():379] send: artifact +2025-01-04 21:28:13,799 INFO SenderThread:2026123 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxOTEyMTE2Ng==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTEzMDY5OQ==', 'versionIndex': 419}}} +2025-01-04 21:28:13,799 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: defer +2025-01-04 21:28:13,799 INFO SenderThread:2026123 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-04 21:28:13,799 INFO SenderThread:2026123 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-04 21:28:14,027 INFO SenderThread:2026123 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:28:14,027 INFO SenderThread:2026123 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files +2025-01-04 21:28:14,028 INFO SenderThread:2026123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/config.yaml config.yaml +2025-01-04 21:28:14,028 INFO SenderThread:2026123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/events.out.tfevents.1736043808.leonmkim-ROG-Strix-G15CS-G15CS.2025958.0 events.out.tfevents.1736043808.leonmkim-ROG-Strix-G15CS-G15CS.2025958.0 +2025-01-04 21:28:14,028 INFO SenderThread:2026123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/wandb-summary.json wandb-summary.json +2025-01-04 21:28:14,028 INFO SenderThread:2026123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/requirements.txt requirements.txt +2025-01-04 21:28:14,028 INFO SenderThread:2026123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/wandb-metadata.json wandb-metadata.json +2025-01-04 21:28:14,028 INFO SenderThread:2026123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log output.log +2025-01-04 21:28:14,028 INFO SenderThread:2026123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/diff.patch diff.patch +2025-01-04 21:28:14,028 INFO SenderThread:2026123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-04 21:28:14,030 INFO SenderThread:2026123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval/1_eval_1_b0947b11fcc3d3b55612.mp4 media/videos/eval/1_eval_1_b0947b11fcc3d3b55612.mp4 +2025-01-04 21:28:14,030 INFO SenderThread:2026123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval/3_eval_3_c975888550477f48ba44.mp4 media/videos/eval/3_eval_3_c975888550477f48ba44.mp4 +2025-01-04 21:28:14,030 INFO SenderThread:2026123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval/4_eval_4_d357aac167e5d7bc548b.mp4 media/videos/eval/4_eval_4_d357aac167e5d7bc548b.mp4 +2025-01-04 21:28:14,030 INFO SenderThread:2026123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval/2_eval_2_8a57ace6d7baf7108296.mp4 media/videos/eval/2_eval_2_8a57ace6d7baf7108296.mp4 +2025-01-04 21:28:14,030 INFO SenderThread:2026123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/videos/eval/0_eval_0_46f66c2ba3db8f12f00c.mp4 media/videos/eval/0_eval_0_46f66c2ba3db8f12f00c.mp4 +2025-01-04 21:28:14,031 INFO SenderThread:2026123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/table/eval/total_success_rate_ci_6_2ba98cbc4fea23f33f14.table.json media/table/eval/total_success_rate_ci_6_2ba98cbc4fea23f33f14.table.json +2025-01-04 21:28:14,031 INFO SenderThread:2026123 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-04 21:28:14,031 INFO SenderThread:2026123 [sender.py:transition_state():614] send defer: 10 +2025-01-04 21:28:14,031 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:28:14,031 INFO HandlerThread:2026123 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-04 21:28:14,037 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: defer +2025-01-04 21:28:14,037 INFO SenderThread:2026123 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-04 21:28:14,037 INFO SenderThread:2026123 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 21:28:14,183 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:28:14,184 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:28:14,234 INFO wandb-upload_1:2026123 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/config.yaml +2025-01-04 21:28:14,238 INFO wandb-upload_2:2026123 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/wandb-summary.json +2025-01-04 21:28:14,265 INFO wandb-upload_3:2026123 [upload_job.py:push():130] Uploaded file /tmp/tmpq2mmia7ewandb/6tlriv4l-events.out.tfevents.1736043808.leonmkim-ROG-Strix-G15CS-G15CS.2025958.0 +2025-01-04 21:28:14,269 INFO wandb-upload_4:2026123 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/requirements.txt +2025-01-04 21:28:14,271 INFO wandb-upload_0:2026123 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/files/output.log +2025-01-04 21:28:14,471 INFO Thread-11 (_thread_body):2026123 [sender.py:transition_state():614] send defer: 11 +2025-01-04 21:28:14,471 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:28:14,472 INFO HandlerThread:2026123 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-04 21:28:14,472 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: defer +2025-01-04 21:28:14,472 INFO SenderThread:2026123 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-04 21:28:14,472 INFO SenderThread:2026123 [file_pusher.py:join():175] waiting for file pusher +2025-01-04 21:28:14,473 INFO SenderThread:2026123 [sender.py:transition_state():614] send defer: 12 +2025-01-04 21:28:14,473 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:28:14,473 INFO HandlerThread:2026123 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-04 21:28:14,473 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: defer +2025-01-04 21:28:14,473 INFO SenderThread:2026123 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-04 21:28:14,473 INFO SenderThread:2026123 [file_stream.py:finish():601] file stream finish called +2025-01-04 21:28:14,670 INFO SenderThread:2026123 [file_stream.py:finish():605] file stream finish is done +2025-01-04 21:28:14,670 INFO SenderThread:2026123 [sender.py:transition_state():614] send defer: 13 +2025-01-04 21:28:14,670 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:28:14,670 INFO HandlerThread:2026123 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-04 21:28:14,670 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: defer +2025-01-04 21:28:14,670 INFO SenderThread:2026123 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-04 21:28:14,671 INFO SenderThread:2026123 [sender.py:transition_state():614] send defer: 14 +2025-01-04 21:28:14,671 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:28:14,671 INFO HandlerThread:2026123 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-04 21:28:14,671 DEBUG SenderThread:2026123 [sender.py:send():379] send: final +2025-01-04 21:28:14,671 DEBUG SenderThread:2026123 [sender.py:send():379] send: footer +2025-01-04 21:28:14,671 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: defer +2025-01-04 21:28:14,671 INFO SenderThread:2026123 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-04 21:28:14,671 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:28:14,672 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:28:14,672 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: server_info +2025-01-04 21:28:14,672 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:28:14,672 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: get_summary +2025-01-04 21:28:14,673 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: server_info +2025-01-04 21:28:14,674 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-04 21:28:14,675 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:28:14,733 DEBUG SenderThread:2026123 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:28:14,734 INFO MainThread:2026123 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-04 21:28:14,734 INFO MainThread:2026123 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-04 21:28:14,734 INFO MainThread:2026123 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-04 21:28:14,735 DEBUG HandlerThread:2026123 [handler.py:handle_request():158] handle_request: shutdown +2025-01-04 21:28:14,735 INFO HandlerThread:2026123 [handler.py:finish():882] shutting down handler +2025-01-04 21:28:15,673 INFO WriterThread:2026123 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/run-chnx0pdg.wandb +2025-01-04 21:28:15,734 INFO SenderThread:2026123 [sender.py:finish():1615] shutting down sender +2025-01-04 21:28:15,734 INFO SenderThread:2026123 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 21:28:15,734 INFO SenderThread:2026123 [file_pusher.py:join():175] waiting for file pusher diff --git a/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/logs/debug.log b/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..70ea78ac90294a3f323f5ccb9944db93f94c89ee --- /dev/null +++ b/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-04 21:23:27,657 INFO MainThread:2025958 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-04 21:23:27,657 INFO MainThread:2025958 [wandb_setup.py:_flush():76] Configure stats pid to 2025958 +2025-01-04 21:23:27,657 INFO MainThread:2025958 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-04 21:23:27,657 INFO MainThread:2025958 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/settings +2025-01-04 21:23:27,657 INFO MainThread:2025958 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-04 21:23:27,657 INFO MainThread:2025958 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-04 21:23:27,657 INFO MainThread:2025958 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-04 21:23:27,657 INFO MainThread:2025958 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-04 21:23:27,657 INFO MainThread:2025958 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/logs/debug.log +2025-01-04 21:23:27,657 INFO MainThread:2025958 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/logs/debug-internal.log +2025-01-04 21:23:27,657 INFO MainThread:2025958 [wandb_init.py:init():569] calling init triggers +2025-01-04 21:23:27,657 INFO MainThread:2025958 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1938_0', 'wandb_notes': '1938_0_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg'} +2025-01-04 21:23:27,658 INFO MainThread:2025958 [wandb_init.py:init():619] starting backend +2025-01-04 21:23:27,658 INFO MainThread:2025958 [wandb_init.py:init():623] setting up manager +2025-01-04 21:23:27,662 INFO MainThread:2025958 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-04 21:23:27,664 INFO MainThread:2025958 [wandb_init.py:init():631] backend started and connected +2025-01-04 21:23:27,675 INFO MainThread:2025958 [wandb_init.py:init():720] updated telemetry +2025-01-04 21:23:27,683 INFO MainThread:2025958 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-04 21:23:27,942 INFO MainThread:2025958 [wandb_run.py:_on_init():2435] communicating current version +2025-01-04 21:23:27,984 INFO MainThread:2025958 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-04 21:23:27,984 INFO MainThread:2025958 [wandb_init.py:init():804] starting run threads in backend +2025-01-04 21:23:28,312 INFO MainThread:2025958 [wandb_run.py:_console_start():2413] atexit reg +2025-01-04 21:23:28,313 INFO MainThread:2025958 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-04 21:23:28,313 INFO MainThread:2025958 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-04 21:23:28,313 INFO MainThread:2025958 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-04 21:23:28,316 INFO MainThread:2025958 [wandb_init.py:init():847] run started, returning control to user process +2025-01-04 21:23:28,316 INFO MainThread:2025958 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/chnx0pdg/tb, True +2025-01-04 21:23:32,405 INFO MainThread:2025958 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-04 21:28:15,735 WARNING MsgRouterThr:2025958 [router.py:message_loop():77] message_loop has been closed diff --git a/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/run-chnx0pdg.wandb b/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/run-chnx0pdg.wandb new file mode 100644 index 0000000000000000000000000000000000000000..8a6dddfef7f964e4cbfc874ea8167b82fa963de7 Binary files /dev/null and b/chnx0pdg/wandb/run-20250104_212327-chnx0pdg/run-chnx0pdg.wandb differ diff --git a/pauhv4zk/.hydra/config.yaml b/pauhv4zk/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..30c3bfdc0a6aec7938f3f2b6ed72878e72cc1f21 --- /dev/null +++ b/pauhv4zk/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1938_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/pauhv4zk/.hydra/hydra.yaml b/pauhv4zk/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f1ac0d9747dbc5ed9cb3be9c4ee272ec0b9f125a --- /dev/null +++ b/pauhv4zk/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/pauhv4zk/.hydra/overrides.yaml b/pauhv4zk/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/pauhv4zk/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/pauhv4zk/episode_rosbags/aligned_depth_to_color_K.npy b/pauhv4zk/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/pauhv4zk/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/pauhv4zk/episode_rosbags/color_K.npy b/pauhv4zk/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/pauhv4zk/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/pauhv4zk/eval_video/0_eval.mp4 b/pauhv4zk/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9092a7b5ba5d5c609e4c172fce7dc8cfdacff008 --- /dev/null +++ b/pauhv4zk/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4895a45f2099a6358d9b28958cd13550f10423ab4c8af10c7a84d1a6082182bb +size 1157651 diff --git a/pauhv4zk/eval_video/1_eval.mp4 b/pauhv4zk/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e7c7e5594c1d682d7b7ae8f2e65305bd539cf528 --- /dev/null +++ b/pauhv4zk/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b53855a2640f91b72a281613afea099032a94909c5cce05b233d80567fe19579 +size 1158998 diff --git a/pauhv4zk/eval_video/2_eval.mp4 b/pauhv4zk/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..99e22db9aa2cebe1729cf92324910317ad2281b4 --- /dev/null +++ b/pauhv4zk/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a282a3c65bafd15a5fd649e78be4615591940c6fa59a006eca48b8c88972e2d +size 1110522 diff --git a/pauhv4zk/eval_video/3_eval.mp4 b/pauhv4zk/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..54b816d5319771712abea239e26605698a585f2e --- /dev/null +++ b/pauhv4zk/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20a3393f5cad2132e7edf766a83eafcafb9865f2b2edf577a98a763bea1ebe2b +size 1138666 diff --git a/pauhv4zk/eval_video/4_eval.mp4 b/pauhv4zk/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cfe3d58c5eb157f059be021f080b343d040706d2 --- /dev/null +++ b/pauhv4zk/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f3ac500e502283fe4f2efdba56f7f6703259b0bb35f585c601ad045a4188d3f +size 1201784 diff --git a/pauhv4zk/tb/events.out.tfevents.1735959951.leonmkim-ROG-Strix-G15CS-G15CS.1978742.0 b/pauhv4zk/tb/events.out.tfevents.1735959951.leonmkim-ROG-Strix-G15CS-G15CS.1978742.0 new file mode 100644 index 0000000000000000000000000000000000000000..24ce3aaea31b0dc1864c65ebc59f6a0ec5610eab --- /dev/null +++ b/pauhv4zk/tb/events.out.tfevents.1735959951.leonmkim-ROG-Strix-G15CS-G15CS.1978742.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8b9d2cd837581c20a202d315b0a2a054bdaa00488d5573da2ca8e8a47b80cfa +size 6539 diff --git a/pauhv4zk/wandb/debug-internal.log b/pauhv4zk/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..465ebb441bb68d3b9cb714c3110d885c8378411b --- /dev/null +++ b/pauhv4zk/wandb/debug-internal.log @@ -0,0 +1,774 @@ +2025-01-03 22:05:50,367 INFO StreamThr :1978913 [internal.py:wandb_internal():85] W&B internal server running at pid: 1978913, started at: 2025-01-03 22:05:50.366945 +2025-01-03 22:05:50,368 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status +2025-01-03 22:05:50,382 INFO WriterThread:1978913 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/run-pauhv4zk.wandb +2025-01-03 22:05:50,382 DEBUG SenderThread:1978913 [sender.py:send():379] send: header +2025-01-03 22:05:50,390 DEBUG SenderThread:1978913 [sender.py:send():379] send: run +2025-01-03 22:05:50,596 INFO SenderThread:1978913 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files +2025-01-03 22:05:50,596 INFO SenderThread:1978913 [sender.py:_start_run_threads():1188] run started: pauhv4zk with start time 1735959950.368894 +2025-01-03 22:05:50,612 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: check_version +2025-01-03 22:05:50,612 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: check_version +2025-01-03 22:05:50,687 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: run_start +2025-01-03 22:05:50,691 DEBUG HandlerThread:1978913 [system_info.py:__init__():26] System info init +2025-01-03 22:05:50,691 DEBUG HandlerThread:1978913 [system_info.py:__init__():41] System info init done +2025-01-03 22:05:50,692 INFO HandlerThread:1978913 [system_monitor.py:start():194] Starting system monitor +2025-01-03 22:05:50,692 INFO SystemMonitor:1978913 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-03 22:05:50,692 INFO HandlerThread:1978913 [system_monitor.py:probe():214] Collecting system info +2025-01-03 22:05:50,692 INFO SystemMonitor:1978913 [interfaces.py:start():188] Started cpu monitoring +2025-01-03 22:05:50,693 INFO SystemMonitor:1978913 [interfaces.py:start():188] Started disk monitoring +2025-01-03 22:05:50,693 INFO SystemMonitor:1978913 [interfaces.py:start():188] Started gpu monitoring +2025-01-03 22:05:50,694 INFO SystemMonitor:1978913 [interfaces.py:start():188] Started memory monitoring +2025-01-03 22:05:50,695 INFO SystemMonitor:1978913 [interfaces.py:start():188] Started network monitoring +2025-01-03 22:05:50,705 DEBUG HandlerThread:1978913 [system_info.py:probe():152] Probing system +2025-01-03 22:05:50,707 DEBUG HandlerThread:1978913 [system_info.py:_probe_git():137] Probing git +2025-01-03 22:05:50,711 DEBUG HandlerThread:1978913 [system_info.py:_probe_git():145] Probing git done +2025-01-03 22:05:50,711 DEBUG HandlerThread:1978913 [system_info.py:probe():200] Probing system done +2025-01-03 22:05:50,711 DEBUG HandlerThread:1978913 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-04T03:05:50.705915', 'startedAt': '2025-01-04T03:05:50.358113', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'c537eb6734155592809c2a7db5cca971ec186d72'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4122.793375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.025, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4497.434, 'min': 800.0, 'max': 4700.0}, {'current': 4496.663, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 818.3720664978027}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-03 22:05:50,712 INFO HandlerThread:1978913 [system_monitor.py:probe():224] Finished collecting system info +2025-01-03 22:05:50,712 INFO HandlerThread:1978913 [system_monitor.py:probe():227] Publishing system info +2025-01-03 22:05:50,712 DEBUG HandlerThread:1978913 [system_info.py:_save_code():44] Saving code +2025-01-03 22:05:50,713 DEBUG HandlerThread:1978913 [system_info.py:_save_code():67] Saving code done +2025-01-03 22:05:50,713 DEBUG HandlerThread:1978913 [system_info.py:_save_patches():84] Saving git patches +2025-01-03 22:05:50,779 DEBUG HandlerThread:1978913 [system_info.py:_save_patches():126] Saving git patches done +2025-01-03 22:05:50,779 INFO HandlerThread:1978913 [system_monitor.py:probe():229] Finished publishing system info +2025-01-03 22:05:50,782 DEBUG SenderThread:1978913 [sender.py:send():379] send: files +2025-01-03 22:05:50,782 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-03 22:05:50,782 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-03 22:05:50,783 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-03 22:05:51,016 INFO wandb-upload_2:1978913 [upload_job.py:push():130] Uploaded file /tmp/tmp3gj0zt0iwandb/88qa0844-diff.patch +2025-01-03 22:05:51,047 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: python_packages +2025-01-03 22:05:51,047 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: python_packages +2025-01-03 22:05:51,048 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:05:51,048 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:05:51,050 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:05:51,050 INFO HandlerThread:1978913 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/tb" + save: true +} +control { + relay_id: "559c5b11046641e18dc89c7d994a8133" +} +_info { + stream_id: "pauhv4zk" +} + +2025-01-03 22:05:51,050 DEBUG HandlerThread:1978913 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-03 22:05:51,078 INFO wandb-upload_0:1978913 [upload_job.py:push():130] Uploaded file /tmp/tmp3gj0zt0iwandb/h3klqk8s-wandb-metadata.json +2025-01-03 22:05:51,097 INFO wandb-upload_1:1978913 [upload_job.py:push():130] Uploaded file /tmp/tmp3gj0zt0iwandb/2nlk63jw-code/FISH/eval_robot.py +2025-01-03 22:05:51,150 DEBUG SenderThread:1978913 [sender.py:send():379] send: telemetry +2025-01-03 22:05:51,151 DEBUG SenderThread:1978913 [sender.py:send():379] send: tbrecord +2025-01-03 22:05:51,151 DEBUG SenderThread:1978913 [sender.py:send():379] send: files +2025-01-03 22:05:51,151 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file events.out.tfevents.1735959951.leonmkim-ROG-Strix-G15CS-G15CS.1978742.0 with policy live +2025-01-03 22:05:51,336 INFO wandb-upload_2:1978913 [upload_job.py:push():130] Uploaded file /tmp/tmp3gj0zt0iwandb/zqz83b5d-events.out.tfevents.1735959951.leonmkim-ROG-Strix-G15CS-G15CS.1978742.0 +2025-01-03 22:05:51,598 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/requirements.txt +2025-01-03 22:05:51,598 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:05:51,598 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/wandb-metadata.json +2025-01-03 22:05:51,598 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/code/FISH/eval_robot.py +2025-01-03 22:05:51,598 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/diff.patch +2025-01-03 22:05:51,598 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/events.out.tfevents.1735959951.leonmkim-ROG-Strix-G15CS-G15CS.1978742.0 +2025-01-03 22:05:51,598 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/code +2025-01-03 22:05:51,598 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/code/FISH +2025-01-03 22:05:52,048 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:05:53,048 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:05:53,597 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:05:54,049 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:05:55,049 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:05:56,048 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:05:56,049 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:05:57,049 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:05:58,049 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:05:59,049 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:00,049 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:01,048 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:06:01,049 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:02,050 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:03,050 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:04,050 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:05,050 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:06,048 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:06:06,048 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:06:06,089 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:06,196 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:06:07,050 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:08,050 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:09,051 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:09,317 DEBUG SenderThread:1978913 [sender.py:send():379] send: config +2025-01-03 22:06:10,051 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:11,051 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:11,602 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:06:12,049 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:06:12,051 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:13,077 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:14,085 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:15,052 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:15,603 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:06:16,052 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:17,052 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:17,797 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:06:18,052 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:19,052 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:19,604 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:06:20,052 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:21,048 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:06:21,048 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:06:21,052 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:21,605 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:06:22,052 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:23,053 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:23,112 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:06:23,605 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/config.yaml +2025-01-03 22:06:24,053 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:25,053 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:25,606 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:06:26,053 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:27,053 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:27,606 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:06:28,053 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:28,581 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:06:29,057 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:29,607 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:06:30,053 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:31,053 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:32,053 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:33,053 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:34,051 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:06:34,053 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:35,054 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:36,048 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:06:36,048 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:06:36,089 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:37,054 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:38,054 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:39,054 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:39,123 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:06:40,054 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:41,054 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:42,055 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:43,055 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:44,055 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:45,050 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:06:45,055 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:46,055 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:47,055 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:48,055 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:49,055 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:50,056 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:50,190 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:06:50,695 DEBUG SystemMonitor:1978913 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-03 22:06:50,696 DEBUG SenderThread:1978913 [sender.py:send():379] send: stats +2025-01-03 22:06:51,050 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:06:51,051 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:06:51,057 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:51,193 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:06:51,194 DEBUG SenderThread:1978913 [sender.py:send():379] send: files +2025-01-03 22:06:51,194 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_4895a45f2099a6358d9b.mp4 with policy now +2025-01-03 22:06:51,194 DEBUG SenderThread:1978913 [sender.py:send():379] send: history +2025-01-03 22:06:51,194 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:06:51,195 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:06:51,473 INFO wandb-upload_1:1978913 [upload_job.py:push():130] Uploaded file /tmp/tmp3gj0zt0iwandb/xox0l4vk-media/videos/eval/0_eval_0_4895a45f2099a6358d9b.mp4 +2025-01-03 22:06:51,612 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:06:51,613 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval/0_eval_0_4895a45f2099a6358d9b.mp4 +2025-01-03 22:06:51,613 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/wandb-summary.json +2025-01-03 22:06:51,613 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval +2025-01-03 22:06:51,613 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media +2025-01-03 22:06:51,613 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos +2025-01-03 22:06:51,873 INFO wandb-upload_2:1978913 [upload_job.py:push():130] Uploaded file /tmp/tmp3gj0zt0iwandb/yf0j5h9j-media/videos/eval/0_eval_0_4895a45f2099a6358d9b.mp4 +2025-01-03 22:06:52,057 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:53,057 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:53,613 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:06:54,057 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:55,060 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:56,009 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:06:56,058 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:57,060 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:57,614 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:06:58,060 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:59,060 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:59,615 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:07:00,060 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:01,052 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:07:01,060 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:01,616 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/events.out.tfevents.1735959951.leonmkim-ROG-Strix-G15CS-G15CS.1978742.0 +2025-01-03 22:07:01,904 INFO wandb-upload_1:1978913 [upload_job.py:push():130] Uploaded file /tmp/tmp3gj0zt0iwandb/h81fxdeo-events.out.tfevents.1735959951.leonmkim-ROG-Strix-G15CS-G15CS.1978742.0 +2025-01-03 22:07:02,060 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:03,061 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:03,617 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:07:04,061 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:05,061 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:05,617 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:07:06,049 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:07:06,049 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:07:06,089 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:06,206 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:07:07,061 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:07,618 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:07:08,061 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:09,061 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:10,064 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:11,061 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:12,052 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:07:12,063 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:13,062 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:14,062 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:15,062 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:16,062 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:17,052 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:07:17,062 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:18,062 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:19,062 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:20,062 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:20,696 DEBUG SenderThread:1978913 [sender.py:send():379] send: stats +2025-01-03 22:07:21,049 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:07:21,049 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:07:21,097 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:22,063 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:22,208 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:07:23,063 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:24,063 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:25,063 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:26,063 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:27,063 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:28,052 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:07:28,063 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:29,064 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:30,064 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:31,064 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:32,064 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:32,950 DEBUG SenderThread:1978913 [sender.py:send():379] send: files +2025-01-03 22:07:32,950 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_b53855a2640f91b72a28.mp4 with policy now +2025-01-03 22:07:32,951 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:07:32,954 DEBUG SenderThread:1978913 [sender.py:send():379] send: history +2025-01-03 22:07:32,954 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:07:32,954 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:07:33,064 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:33,217 INFO wandb-upload_2:1978913 [upload_job.py:push():130] Uploaded file /tmp/tmp3gj0zt0iwandb/edye93xv-media/videos/eval/1_eval_1_b53855a2640f91b72a28.mp4 +2025-01-03 22:07:33,626 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/wandb-summary.json +2025-01-03 22:07:33,626 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:07:33,626 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval/1_eval_1_b53855a2640f91b72a28.mp4 +2025-01-03 22:07:33,627 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval +2025-01-03 22:07:33,919 INFO wandb-upload_1:1978913 [upload_job.py:push():130] Uploaded file /tmp/tmp3gj0zt0iwandb/c1bote73-media/videos/eval/1_eval_1_b53855a2640f91b72a28.mp4 +2025-01-03 22:07:33,966 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:07:34,069 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:35,081 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:35,627 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:07:36,054 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:07:36,054 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:07:36,097 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:37,081 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:37,627 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:07:38,077 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:39,080 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:39,506 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:07:39,628 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:07:40,075 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:41,075 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:41,629 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:07:42,076 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:43,076 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:44,076 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:45,055 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:07:45,076 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:46,076 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:47,076 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:47,473 DEBUG SenderThread:1978913 [sender.py:send():379] send: history +2025-01-03 22:07:47,473 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:07:47,473 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:07:47,631 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/wandb-summary.json +2025-01-03 22:07:47,631 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/events.out.tfevents.1735959951.leonmkim-ROG-Strix-G15CS-G15CS.1978742.0 +2025-01-03 22:07:48,078 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:49,077 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:49,632 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:07:50,077 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:50,587 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:07:50,697 DEBUG SenderThread:1978913 [sender.py:send():379] send: stats +2025-01-03 22:07:51,051 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:07:51,052 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:07:51,077 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:51,632 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:07:52,077 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:53,077 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:53,633 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:07:54,077 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:55,077 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:56,057 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:07:56,077 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:57,077 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:58,077 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:59,081 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:00,079 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:01,058 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:08:01,079 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:02,080 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:03,080 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:04,086 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:05,086 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:06,052 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:08:06,052 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:08:06,093 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:06,135 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:08:07,098 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:08,086 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:09,086 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:10,087 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:11,087 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:12,058 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:08:12,087 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:13,087 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:13,639 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:08:14,087 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:15,087 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:16,092 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:17,058 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:08:17,101 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:18,103 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:18,788 DEBUG SenderThread:1978913 [sender.py:send():379] send: files +2025-01-03 22:08:18,788 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_0a282a3c65bafd15a5fd.mp4 with policy now +2025-01-03 22:08:18,796 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:08:18,797 DEBUG SenderThread:1978913 [sender.py:send():379] send: history +2025-01-03 22:08:18,797 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:08:18,797 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:08:19,050 INFO wandb-upload_2:1978913 [upload_job.py:push():130] Uploaded file /tmp/tmp3gj0zt0iwandb/2bkqasp9-media/videos/eval/2_eval_2_0a282a3c65bafd15a5fd.mp4 +2025-01-03 22:08:19,108 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:19,641 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/wandb-summary.json +2025-01-03 22:08:19,641 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:08:19,641 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval/2_eval_2_0a282a3c65bafd15a5fd.mp4 +2025-01-03 22:08:19,642 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval +2025-01-03 22:08:19,915 INFO wandb-upload_1:1978913 [upload_job.py:push():130] Uploaded file /tmp/tmp3gj0zt0iwandb/lhfrs9d4-media/videos/eval/2_eval_2_0a282a3c65bafd15a5fd.mp4 +2025-01-03 22:08:20,106 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:20,697 DEBUG SenderThread:1978913 [sender.py:send():379] send: stats +2025-01-03 22:08:21,057 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:08:21,058 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:08:21,110 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:21,642 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:08:22,111 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:22,150 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:08:23,108 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:23,643 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:08:24,112 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:25,113 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:26,115 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:27,114 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:27,644 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:08:27,747 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:08:28,115 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:28,303 DEBUG SenderThread:1978913 [sender.py:send():379] send: history +2025-01-03 22:08:28,303 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:08:28,309 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:08:28,644 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/wandb-summary.json +2025-01-03 22:08:28,644 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/events.out.tfevents.1735959951.leonmkim-ROG-Strix-G15CS-G15CS.1978742.0 +2025-01-03 22:08:28,888 INFO wandb-upload_2:1978913 [upload_job.py:push():130] Uploaded file /tmp/tmp3gj0zt0iwandb/yghzxc38-events.out.tfevents.1735959951.leonmkim-ROG-Strix-G15CS-G15CS.1978742.0 +2025-01-03 22:08:29,119 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:29,645 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:08:30,122 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:31,132 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:31,645 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:08:32,119 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:33,019 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:08:33,119 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:33,646 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:08:34,119 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:35,119 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:36,053 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:08:36,054 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:08:36,119 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:37,120 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:38,120 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:38,213 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:08:39,120 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:40,120 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:41,120 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:42,120 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:43,120 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:44,069 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:08:44,120 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:45,121 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:46,120 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:47,121 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:47,651 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:08:48,121 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:49,069 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:08:49,121 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:50,121 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:50,698 DEBUG SenderThread:1978913 [sender.py:send():379] send: stats +2025-01-03 22:08:51,054 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:08:51,054 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:08:51,121 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:52,122 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:53,122 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:53,653 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:08:54,122 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:54,351 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:08:55,122 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:55,654 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:08:56,122 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:57,122 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:58,122 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:59,124 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:59,561 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:08:59,655 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:09:00,039 DEBUG SenderThread:1978913 [sender.py:send():379] send: files +2025-01-03 22:09:00,039 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_20a3393f5cad2132e7ed.mp4 with policy now +2025-01-03 22:09:00,041 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:09:00,042 DEBUG SenderThread:1978913 [sender.py:send():379] send: history +2025-01-03 22:09:00,042 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:09:00,042 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:09:00,124 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:00,307 INFO wandb-upload_1:1978913 [upload_job.py:push():130] Uploaded file /tmp/tmp3gj0zt0iwandb/687zeufw-media/videos/eval/3_eval_3_20a3393f5cad2132e7ed.mp4 +2025-01-03 22:09:00,655 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/wandb-summary.json +2025-01-03 22:09:00,655 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval/3_eval_3_20a3393f5cad2132e7ed.mp4 +2025-01-03 22:09:00,656 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval +2025-01-03 22:09:00,948 INFO wandb-upload_2:1978913 [upload_job.py:push():130] Uploaded file /tmp/tmp3gj0zt0iwandb/fw8qj3ez-media/videos/eval/3_eval_3_20a3393f5cad2132e7ed.mp4 +2025-01-03 22:09:01,137 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:01,656 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:09:02,138 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:03,133 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:04,140 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:04,917 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:09:05,139 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:05,657 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:09:06,057 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:09:06,057 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:09:06,141 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:07,132 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:07,658 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:09:08,136 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:09,137 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:10,136 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:10,275 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:09:10,358 DEBUG SenderThread:1978913 [sender.py:send():379] send: history +2025-01-03 22:09:10,358 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:09:10,359 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:09:10,659 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/wandb-summary.json +2025-01-03 22:09:10,659 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/events.out.tfevents.1735959951.leonmkim-ROG-Strix-G15CS-G15CS.1978742.0 +2025-01-03 22:09:11,136 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:11,659 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:09:12,150 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:13,144 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:13,660 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:09:14,153 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:15,141 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:15,391 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:09:15,660 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:09:16,139 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:17,139 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:17,661 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:09:18,139 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:19,139 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:20,139 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:20,698 DEBUG SenderThread:1978913 [sender.py:send():379] send: stats +2025-01-03 22:09:20,700 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:09:21,055 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:09:21,055 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:09:21,139 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:22,139 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:23,139 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:24,140 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:25,140 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:26,072 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:09:26,140 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:27,140 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:28,140 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:29,140 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:30,140 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:31,072 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:09:31,141 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:32,141 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:33,141 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:34,141 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:35,141 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:36,055 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:09:36,055 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:09:36,141 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:36,207 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:09:37,141 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:38,141 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:39,141 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:40,142 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:41,142 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:42,072 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:09:42,142 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:43,145 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:43,670 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:09:43,991 DEBUG SenderThread:1978913 [sender.py:send():379] send: files +2025-01-03 22:09:43,991 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_9f3ac500e502283fe4f2.mp4 with policy now +2025-01-03 22:09:43,995 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:09:43,996 DEBUG SenderThread:1978913 [sender.py:send():379] send: history +2025-01-03 22:09:43,996 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:09:43,996 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:09:44,150 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:44,251 INFO wandb-upload_1:1978913 [upload_job.py:push():130] Uploaded file /tmp/tmp3gj0zt0iwandb/fqs0z2x1-media/videos/eval/4_eval_4_9f3ac500e502283fe4f2.mp4 +2025-01-03 22:09:44,670 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/wandb-summary.json +2025-01-03 22:09:44,670 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval/4_eval_4_9f3ac500e502283fe4f2.mp4 +2025-01-03 22:09:44,671 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval +2025-01-03 22:09:44,977 INFO wandb-upload_2:1978913 [upload_job.py:push():130] Uploaded file /tmp/tmp3gj0zt0iwandb/syesmre4-media/videos/eval/4_eval_4_9f3ac500e502283fe4f2.mp4 +2025-01-03 22:09:45,153 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:45,671 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:09:46,150 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:47,088 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:09:47,158 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:48,159 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:49,156 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:49,672 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:09:50,152 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:50,698 DEBUG SenderThread:1978913 [sender.py:send():379] send: stats +2025-01-03 22:09:51,064 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:09:51,071 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:09:51,157 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:51,673 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:09:52,164 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:52,369 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:09:52,673 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/events.out.tfevents.1735959951.leonmkim-ROG-Strix-G15CS-G15CS.1978742.0 +2025-01-03 22:09:52,936 INFO wandb-upload_1:1978913 [upload_job.py:push():130] Uploaded file /tmp/tmp3gj0zt0iwandb/p08qcqy3-events.out.tfevents.1735959951.leonmkim-ROG-Strix-G15CS-G15CS.1978742.0 +2025-01-03 22:09:53,156 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:53,412 DEBUG SenderThread:1978913 [sender.py:send():379] send: history +2025-01-03 22:09:53,412 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:09:53,413 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:09:53,609 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 22:09:53,609 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 22:09:53,673 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:09:53,674 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/wandb-summary.json +2025-01-03 22:09:53,674 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/events.out.tfevents.1735959951.leonmkim-ROG-Strix-G15CS-G15CS.1978742.0 +2025-01-03 22:09:53,674 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 22:09:53,674 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media +2025-01-03 22:09:53,674 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/table/eval +2025-01-03 22:09:53,674 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/table +2025-01-03 22:09:53,802 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:09:53,854 INFO wandb-upload_2:1978913 [upload_job.py:push():130] Uploaded file /tmp/tmp3gj0zt0iwandb/ml3llhfk-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 22:09:54,156 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:54,159 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 22:09:54,201 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:09:54,222 INFO wandb-upload_1:1978913 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpjt3eh8h1 +2025-01-03 22:09:54,592 INFO SenderThread:1978913 [sender.py:send_request_log_artifact():1525] logged artifact run-pauhv4zk-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzk1NTM4MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY4NDQ1OQ==', 'latestArtifact': None}} +2025-01-03 22:09:54,592 DEBUG SenderThread:1978913 [sender.py:send():379] send: files +2025-01-03 22:09:54,592 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-03 22:09:54,592 DEBUG SenderThread:1978913 [sender.py:send():379] send: history +2025-01-03 22:09:54,593 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:09:54,593 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:09:54,593 DEBUG SenderThread:1978913 [sender.py:send():379] send: telemetry +2025-01-03 22:09:54,593 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:09:54,597 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:09:54,598 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:09:54,598 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:09:54,598 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:09:54,598 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:09:54,599 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:09:54,599 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:09:54,599 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:09:54,599 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:09:54,600 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:09:54,600 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:09:54,600 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:09:54,600 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:09:54,601 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:09:54,601 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:09:54,601 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:09:54,601 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:09:54,601 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:09:54,602 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:09:54,602 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:09:54,602 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:09:54,602 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 22:09:54,674 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/wandb-summary.json +2025-01-03 22:09:54,674 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/events.out.tfevents.1735959951.leonmkim-ROG-Strix-G15CS-G15CS.1978742.0 +2025-01-03 22:09:54,674 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/table/eval/total_success_rate_ci_6_7fc509d15829a5499a64.table.json +2025-01-03 22:09:54,674 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/table/eval +2025-01-03 22:09:54,828 INFO wandb-upload_2:1978913 [upload_job.py:push():130] Uploaded file /tmp/tmp3gj0zt0iwandb/awjizvt9-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 22:09:54,891 INFO wandb-upload_1:1978913 [upload_job.py:push():130] Uploaded file /tmp/tmp3gj0zt0iwandb/qkz34qym-media/table/eval/total_success_rate_ci_6_7fc509d15829a5499a64.table.json +2025-01-03 22:09:55,001 INFO wandb-upload_0:1978913 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpt85cl7oz +2025-01-03 22:09:55,156 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:55,365 INFO SenderThread:1978913 [sender.py:send_request_log_artifact():1525] logged artifact run-pauhv4zk-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzk1NTM5Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY4NDQ2Mg==', 'latestArtifact': None}} +2025-01-03 22:09:55,365 DEBUG SenderThread:1978913 [sender.py:send():379] send: files +2025-01-03 22:09:55,365 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_7fc509d15829a5499a64.table.json with policy now +2025-01-03 22:09:55,365 DEBUG SenderThread:1978913 [sender.py:send():379] send: history +2025-01-03 22:09:55,366 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:09:55,366 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:09:55,630 INFO wandb-upload_2:1978913 [upload_job.py:push():130] Uploaded file /tmp/tmp3gj0zt0iwandb/gltmt24r-media/table/eval/total_success_rate_ci_6_7fc509d15829a5499a64.table.json +2025-01-03 22:09:55,674 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/wandb-summary.json +2025-01-03 22:09:55,674 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:09:56,156 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:57,157 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:58,077 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:09:58,157 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:59,157 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:00,157 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:01,157 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:02,157 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:03,157 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:04,077 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:10:04,157 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:05,158 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:06,055 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:10:06,055 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:10:06,158 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:07,158 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:08,158 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:09,123 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:10:09,158 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:09,680 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/config.yaml +2025-01-03 22:10:10,158 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:11,159 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:12,159 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:13,159 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:14,159 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:15,025 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:10:15,069 DEBUG SenderThread:1978913 [sender.py:send():379] send: exit +2025-01-03 22:10:15,070 INFO SenderThread:1978913 [sender.py:send_exit():586] handling exit code: 0 +2025-01-03 22:10:15,070 INFO SenderThread:1978913 [sender.py:send_exit():588] handling runtime: 264 +2025-01-03 22:10:15,070 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:10:15,070 INFO SenderThread:1978913 [sender.py:send_exit():594] send defer +2025-01-03 22:10:15,070 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:10:15,070 INFO HandlerThread:1978913 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-03 22:10:15,070 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: defer +2025-01-03 22:10:15,070 INFO SenderThread:1978913 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-03 22:10:15,071 INFO SenderThread:1978913 [sender.py:transition_state():614] send defer: 1 +2025-01-03 22:10:15,071 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:10:15,071 INFO HandlerThread:1978913 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-03 22:10:15,071 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: defer +2025-01-03 22:10:15,071 INFO SenderThread:1978913 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-03 22:10:15,071 INFO SenderThread:1978913 [sender.py:transition_state():614] send defer: 2 +2025-01-03 22:10:15,071 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:10:15,071 INFO HandlerThread:1978913 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-03 22:10:15,071 INFO HandlerThread:1978913 [system_monitor.py:finish():203] Stopping system monitor +2025-01-03 22:10:15,071 DEBUG SystemMonitor:1978913 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-03 22:10:15,071 DEBUG SystemMonitor:1978913 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-03 22:10:15,072 INFO HandlerThread:1978913 [interfaces.py:finish():200] Joined cpu monitor +2025-01-03 22:10:15,072 INFO HandlerThread:1978913 [interfaces.py:finish():200] Joined disk monitor +2025-01-03 22:10:15,125 INFO HandlerThread:1978913 [interfaces.py:finish():200] Joined gpu monitor +2025-01-03 22:10:15,125 INFO HandlerThread:1978913 [interfaces.py:finish():200] Joined memory monitor +2025-01-03 22:10:15,126 INFO HandlerThread:1978913 [interfaces.py:finish():200] Joined network monitor +2025-01-03 22:10:15,126 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: defer +2025-01-03 22:10:15,126 INFO SenderThread:1978913 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-03 22:10:15,126 INFO SenderThread:1978913 [sender.py:transition_state():614] send defer: 3 +2025-01-03 22:10:15,126 DEBUG SenderThread:1978913 [sender.py:send():379] send: stats +2025-01-03 22:10:15,126 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:10:15,126 INFO HandlerThread:1978913 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-03 22:10:15,127 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: defer +2025-01-03 22:10:15,127 INFO SenderThread:1978913 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-03 22:10:15,127 INFO SenderThread:1978913 [sender.py:transition_state():614] send defer: 4 +2025-01-03 22:10:15,127 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:10:15,127 INFO HandlerThread:1978913 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-03 22:10:15,682 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/wandb-summary.json +2025-01-03 22:10:17,683 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:10:21,419 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:10:21,419 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:10:21,419 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 22:10:21,421 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: defer +2025-01-03 22:10:21,421 INFO SenderThread:1978913 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-03 22:10:21,421 INFO SenderThread:1978913 [sender.py:transition_state():614] send defer: 5 +2025-01-03 22:10:21,421 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:10:21,421 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:10:21,421 INFO HandlerThread:1978913 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-03 22:10:21,422 DEBUG SenderThread:1978913 [sender.py:send():379] send: history +2025-01-03 22:10:21,422 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:10:21,422 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:10:21,423 DEBUG SenderThread:1978913 [sender.py:send():379] send: summary +2025-01-03 22:10:21,427 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:10:21,427 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: defer +2025-01-03 22:10:21,427 INFO SenderThread:1978913 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-03 22:10:21,427 INFO SenderThread:1978913 [sender.py:transition_state():614] send defer: 6 +2025-01-03 22:10:21,427 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:10:21,427 INFO HandlerThread:1978913 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-03 22:10:21,427 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: defer +2025-01-03 22:10:21,427 INFO SenderThread:1978913 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-03 22:10:21,427 INFO SenderThread:1978913 [sender.py:transition_state():614] send defer: 7 +2025-01-03 22:10:21,427 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:10:21,428 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:10:21,428 INFO HandlerThread:1978913 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-03 22:10:21,428 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: defer +2025-01-03 22:10:21,428 INFO SenderThread:1978913 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-03 22:10:21,685 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/wandb-summary.json +2025-01-03 22:10:22,071 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:10:23,033 INFO SenderThread:1978913 [sender.py:transition_state():614] send defer: 8 +2025-01-03 22:10:23,033 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:10:23,033 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:10:23,034 INFO HandlerThread:1978913 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-03 22:10:23,034 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: defer +2025-01-03 22:10:23,034 INFO SenderThread:1978913 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-03 22:10:23,034 INFO SenderThread:1978913 [job_builder.py:build():440] Attempting to build job artifact +2025-01-03 22:10:23,036 INFO SenderThread:1978913 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-03 22:10:23,038 INFO SenderThread:1978913 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-03 22:10:23,048 INFO SenderThread:1978913 [sender.py:transition_state():614] send defer: 9 +2025-01-03 22:10:23,048 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:10:23,048 INFO HandlerThread:1978913 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-03 22:10:23,048 DEBUG SenderThread:1978913 [sender.py:send():379] send: artifact +2025-01-03 22:10:23,071 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:10:23,397 INFO wandb-upload_1:1978913 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpjj1azryb +2025-01-03 22:10:23,397 INFO wandb-upload_2:1978913 [upload_job.py:push():85] Skipped uploading /tmp/tmp31jrvtyv/wandb-job.json +2025-01-03 22:10:23,507 INFO wandb-upload_0:1978913 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpn9gg5cm_ +2025-01-03 22:10:23,685 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:10:24,100 INFO SenderThread:1978913 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNzk1NTgzMA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNzkwNDU3OA==', 'versionIndex': 410}}} +2025-01-03 22:10:24,100 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: defer +2025-01-03 22:10:24,100 INFO SenderThread:1978913 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-03 22:10:24,100 INFO SenderThread:1978913 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-03 22:10:24,686 INFO SenderThread:1978913 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files +2025-01-03 22:10:24,686 INFO SenderThread:1978913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/config.yaml config.yaml +2025-01-03 22:10:24,686 INFO SenderThread:1978913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/wandb-summary.json wandb-summary.json +2025-01-03 22:10:24,687 INFO SenderThread:1978913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/events.out.tfevents.1735959951.leonmkim-ROG-Strix-G15CS-G15CS.1978742.0 events.out.tfevents.1735959951.leonmkim-ROG-Strix-G15CS-G15CS.1978742.0 +2025-01-03 22:10:24,687 INFO SenderThread:1978913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/requirements.txt requirements.txt +2025-01-03 22:10:24,687 INFO SenderThread:1978913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/wandb-metadata.json wandb-metadata.json +2025-01-03 22:10:24,687 INFO SenderThread:1978913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log output.log +2025-01-03 22:10:24,687 INFO SenderThread:1978913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/diff.patch diff.patch +2025-01-03 22:10:24,691 INFO SenderThread:1978913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-03 22:10:24,691 INFO SenderThread:1978913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval/0_eval_0_4895a45f2099a6358d9b.mp4 media/videos/eval/0_eval_0_4895a45f2099a6358d9b.mp4 +2025-01-03 22:10:24,691 INFO SenderThread:1978913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval/1_eval_1_b53855a2640f91b72a28.mp4 media/videos/eval/1_eval_1_b53855a2640f91b72a28.mp4 +2025-01-03 22:10:24,691 INFO SenderThread:1978913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval/4_eval_4_9f3ac500e502283fe4f2.mp4 media/videos/eval/4_eval_4_9f3ac500e502283fe4f2.mp4 +2025-01-03 22:10:24,691 INFO SenderThread:1978913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval/3_eval_3_20a3393f5cad2132e7ed.mp4 media/videos/eval/3_eval_3_20a3393f5cad2132e7ed.mp4 +2025-01-03 22:10:24,691 INFO SenderThread:1978913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval/2_eval_2_0a282a3c65bafd15a5fd.mp4 media/videos/eval/2_eval_2_0a282a3c65bafd15a5fd.mp4 +2025-01-03 22:10:24,693 INFO SenderThread:1978913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/table/eval/total_success_rate_ci_6_7fc509d15829a5499a64.table.json media/table/eval/total_success_rate_ci_6_7fc509d15829a5499a64.table.json +2025-01-03 22:10:24,693 INFO SenderThread:1978913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 22:10:24,693 INFO SenderThread:1978913 [sender.py:transition_state():614] send defer: 10 +2025-01-03 22:10:24,693 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:10:24,696 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:10:24,696 INFO HandlerThread:1978913 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-03 22:10:24,696 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: defer +2025-01-03 22:10:24,696 INFO SenderThread:1978913 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-03 22:10:24,696 INFO SenderThread:1978913 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 22:10:24,878 INFO wandb-upload_1:1978913 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/config.yaml +2025-01-03 22:10:24,891 INFO wandb-upload_4:1978913 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/requirements.txt +2025-01-03 22:10:24,896 INFO wandb-upload_2:1978913 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/wandb-summary.json +2025-01-03 22:10:24,929 INFO wandb-upload_0:1978913 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:10:24,932 INFO wandb-upload_3:1978913 [upload_job.py:push():130] Uploaded file /tmp/tmp3gj0zt0iwandb/6nvkzf8o-events.out.tfevents.1735959951.leonmkim-ROG-Strix-G15CS-G15CS.1978742.0 +2025-01-03 22:10:25,072 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:10:25,072 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:10:25,133 INFO Thread-11 (_thread_body):1978913 [sender.py:transition_state():614] send defer: 11 +2025-01-03 22:10:25,133 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:10:25,133 INFO HandlerThread:1978913 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-03 22:10:25,133 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: defer +2025-01-03 22:10:25,133 INFO SenderThread:1978913 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-03 22:10:25,133 INFO SenderThread:1978913 [file_pusher.py:join():175] waiting for file pusher +2025-01-03 22:10:25,135 INFO SenderThread:1978913 [sender.py:transition_state():614] send defer: 12 +2025-01-03 22:10:25,135 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:10:25,135 INFO HandlerThread:1978913 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-03 22:10:25,135 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: defer +2025-01-03 22:10:25,135 INFO SenderThread:1978913 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-03 22:10:25,135 INFO SenderThread:1978913 [file_stream.py:finish():601] file stream finish called +2025-01-03 22:10:25,334 INFO SenderThread:1978913 [file_stream.py:finish():605] file stream finish is done +2025-01-03 22:10:25,334 INFO SenderThread:1978913 [sender.py:transition_state():614] send defer: 13 +2025-01-03 22:10:25,334 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:10:25,334 INFO HandlerThread:1978913 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-03 22:10:25,334 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: defer +2025-01-03 22:10:25,335 INFO SenderThread:1978913 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-03 22:10:25,335 INFO SenderThread:1978913 [sender.py:transition_state():614] send defer: 14 +2025-01-03 22:10:25,335 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:10:25,335 INFO HandlerThread:1978913 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-03 22:10:25,335 DEBUG SenderThread:1978913 [sender.py:send():379] send: final +2025-01-03 22:10:25,335 DEBUG SenderThread:1978913 [sender.py:send():379] send: footer +2025-01-03 22:10:25,335 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: defer +2025-01-03 22:10:25,335 INFO SenderThread:1978913 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-03 22:10:25,336 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:10:25,336 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:10:25,336 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: server_info +2025-01-03 22:10:25,336 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:10:25,336 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: get_summary +2025-01-03 22:10:25,337 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-03 22:10:25,337 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: server_info +2025-01-03 22:10:25,339 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:25,384 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:10:25,384 INFO MainThread:1978913 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-03 22:10:25,385 INFO MainThread:1978913 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-03 22:10:25,385 INFO MainThread:1978913 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-03 22:10:25,385 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: shutdown +2025-01-03 22:10:25,385 INFO HandlerThread:1978913 [handler.py:finish():882] shutting down handler +2025-01-03 22:10:26,337 INFO WriterThread:1978913 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/run-pauhv4zk.wandb +2025-01-03 22:10:26,384 INFO SenderThread:1978913 [sender.py:finish():1615] shutting down sender +2025-01-03 22:10:26,385 INFO SenderThread:1978913 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 22:10:26,385 INFO SenderThread:1978913 [file_pusher.py:join():175] waiting for file pusher diff --git a/pauhv4zk/wandb/debug.log b/pauhv4zk/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..3fa42ae73507a62cffec551eae9e8e988be7f946 --- /dev/null +++ b/pauhv4zk/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-03 22:05:50,359 INFO MainThread:1978742 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-03 22:05:50,359 INFO MainThread:1978742 [wandb_setup.py:_flush():76] Configure stats pid to 1978742 +2025-01-03 22:05:50,359 INFO MainThread:1978742 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-03 22:05:50,359 INFO MainThread:1978742 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/settings +2025-01-03 22:05:50,359 INFO MainThread:1978742 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-03 22:05:50,359 INFO MainThread:1978742 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-03 22:05:50,359 INFO MainThread:1978742 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-03 22:05:50,359 INFO MainThread:1978742 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-03 22:05:50,359 INFO MainThread:1978742 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/logs/debug.log +2025-01-03 22:05:50,359 INFO MainThread:1978742 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/logs/debug-internal.log +2025-01-03 22:05:50,359 INFO MainThread:1978742 [wandb_init.py:init():569] calling init triggers +2025-01-03 22:05:50,360 INFO MainThread:1978742 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1938_0', 'wandb_notes': '1938_0_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk'} +2025-01-03 22:05:50,360 INFO MainThread:1978742 [wandb_init.py:init():619] starting backend +2025-01-03 22:05:50,361 INFO MainThread:1978742 [wandb_init.py:init():623] setting up manager +2025-01-03 22:05:50,366 INFO MainThread:1978742 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-03 22:05:50,368 INFO MainThread:1978742 [wandb_init.py:init():631] backend started and connected +2025-01-03 22:05:50,381 INFO MainThread:1978742 [wandb_init.py:init():720] updated telemetry +2025-01-03 22:05:50,388 INFO MainThread:1978742 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-03 22:05:50,612 INFO MainThread:1978742 [wandb_run.py:_on_init():2435] communicating current version +2025-01-03 22:05:50,682 INFO MainThread:1978742 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-03 22:05:50,682 INFO MainThread:1978742 [wandb_init.py:init():804] starting run threads in backend +2025-01-03 22:05:51,048 INFO MainThread:1978742 [wandb_run.py:_console_start():2413] atexit reg +2025-01-03 22:05:51,048 INFO MainThread:1978742 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-03 22:05:51,048 INFO MainThread:1978742 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-03 22:05:51,048 INFO MainThread:1978742 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-03 22:05:51,050 INFO MainThread:1978742 [wandb_init.py:init():847] run started, returning control to user process +2025-01-03 22:05:51,050 INFO MainThread:1978742 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/tb, True +2025-01-03 22:06:09,316 INFO MainThread:1978742 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'} +2025-01-03 22:10:26,386 WARNING MsgRouterThr:1978742 [router.py:message_loop():77] message_loop has been closed diff --git a/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/code/FISH/eval_robot.py b/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/config.yaml b/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d7ea12dcfd29e8fadd647b1c2e686b535e1f4820 --- /dev/null +++ b/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 1 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1938_0' +wandb_notes: + desc: null + value: 1938_0_restarted_5 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735959950 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: modelsys +left_book_slot: + desc: null + value: twodim diff --git a/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/diff.patch b/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..77c852fe9f419930a0fb76a7914e3aa04a6cbbb8 --- /dev/null +++ b/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/diff.patch @@ -0,0 +1,57 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 57a851f..06bbd87 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -138,12 +138,19 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' ++# wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1922_2' # seed 2 ++wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1006_0' ++# wandb_run_id: '1926_1' # seed 1 ++# wandb_run_id: '1930_0' # dataset shuffle seed 1 ++ ++ + + # RGBD+mask+contact(contact+DTC)+act history +-wandb_run_id: '1745_0' ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 71a5d1c..c7eadce 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -159,8 +159,10 @@ run_id_list = [ + ] + + run_id_list = [ +- # '1006_0', +- '1745_0' ++ # '1922_1', ++ # '1922_2', ++ '1926_1', ++ '1930_0', + ] + + checkpoint_epoch = 12000 +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index 6f97e26..285afc8 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,8 +330,8 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.005 +- self.move_to_rot_tolerance = 1.0 ++ self.move_to_trans_tolerance = 0.01 ++ self.move_to_rot_tolerance = 2.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) + self.move_to_target_timeout = 5.0 diff --git a/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json b/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json new file mode 100644 index 0000000000000000000000000000000000000000..abffe612675a3f5ba7b32314013cd92e261fa3b0 --- /dev/null +++ b/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.449680686007073, 1, 0.47999376058978616, 1, 0.6638181209236573, 1, 1, 1]]} \ No newline at end of file diff --git a/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/table/eval/total_success_rate_ci_6_7fc509d15829a5499a64.table.json b/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/table/eval/total_success_rate_ci_6_7fc509d15829a5499a64.table.json new file mode 100644 index 0000000000000000000000000000000000000000..82a5168cfc45f90de727ff09cfc07e879044d8ac --- /dev/null +++ b/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/table/eval/total_success_rate_ci_6_7fc509d15829a5499a64.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[14, 15, 0.9333333333333333, 0.6518780588799942, 0.998813567583338, 0.7327057420742016, 0.9982055815802992, 0.7706979363615327, 0.9904621317001749, 0.7770464345258705, 0.9919731914013155]]} \ No newline at end of file diff --git a/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval/0_eval_0_4895a45f2099a6358d9b.mp4 b/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval/0_eval_0_4895a45f2099a6358d9b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9092a7b5ba5d5c609e4c172fce7dc8cfdacff008 --- /dev/null +++ b/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval/0_eval_0_4895a45f2099a6358d9b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4895a45f2099a6358d9b28958cd13550f10423ab4c8af10c7a84d1a6082182bb +size 1157651 diff --git a/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval/1_eval_1_b53855a2640f91b72a28.mp4 b/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval/1_eval_1_b53855a2640f91b72a28.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e7c7e5594c1d682d7b7ae8f2e65305bd539cf528 --- /dev/null +++ b/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval/1_eval_1_b53855a2640f91b72a28.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b53855a2640f91b72a281613afea099032a94909c5cce05b233d80567fe19579 +size 1158998 diff --git a/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval/2_eval_2_0a282a3c65bafd15a5fd.mp4 b/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval/2_eval_2_0a282a3c65bafd15a5fd.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..99e22db9aa2cebe1729cf92324910317ad2281b4 --- /dev/null +++ b/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval/2_eval_2_0a282a3c65bafd15a5fd.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a282a3c65bafd15a5fd649e78be4615591940c6fa59a006eca48b8c88972e2d +size 1110522 diff --git a/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval/3_eval_3_20a3393f5cad2132e7ed.mp4 b/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval/3_eval_3_20a3393f5cad2132e7ed.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..54b816d5319771712abea239e26605698a585f2e --- /dev/null +++ b/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval/3_eval_3_20a3393f5cad2132e7ed.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20a3393f5cad2132e7edf766a83eafcafb9865f2b2edf577a98a763bea1ebe2b +size 1138666 diff --git a/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval/4_eval_4_9f3ac500e502283fe4f2.mp4 b/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval/4_eval_4_9f3ac500e502283fe4f2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cfe3d58c5eb157f059be021f080b343d040706d2 --- /dev/null +++ b/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval/4_eval_4_9f3ac500e502283fe4f2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f3ac500e502283fe4f2efdba56f7f6703259b0bb35f585c601ad045a4188d3f +size 1201784 diff --git a/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log b/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..ee66a9ffca13ba742afc89c351732abc1c444bc0 --- /dev/null +++ b/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log @@ -0,0 +1,281 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735959969.317511]: resetting environment +[INFO] [1735959969.322103]: cleared current plan +[INFO] [1735959969.323360]: moving to home +[INFO] [1735959970.325265]: reached home +[INFO] [1735959970.327603]: reset action history +[INFO] [1735959973.034658]: environment reset +Starting episode 0 +[INFO] [1735959973.036733]: resetting environment +[INFO] [1735959973.039572]: cleared current plan +[INFO] [1735959973.041072]: moving to home +[INFO] [1735959974.043722]: reached home +[INFO] [1735959974.045549]: reset action history +[INFO] [1735959976.753336]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1735959979.476586]: resetting environment +[INFO] [1735959979.482176]: cleared current plan +[INFO] [1735959979.483310]: moving to home +[INFO] [1735959980.485215]: reached home +[INFO] [1735959980.486749]: reset action history +[INFO] [1735959983.193997]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/episode_rosbags/episode_0_2025-01-03-22-06-23.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1978742.log +started roslaunch server http://158.130.50.37:35535/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1979040] +process[print_text-2]: started with pid [1979041] +process[rosbag_record-3]: started with pid [1979042] +started bagging! +For topic gripper_width: timestamp difference is 108343959 for nearest: 1735959983828216195 - target: 1735959983719872236 at idx 8 +For topic gripper_width: timestamp difference is 109760938 for nearest: 1735959985561548887 - target: 1735959985451787949 at idx 28 +For topic gripper_width: timestamp difference is 110087571 for nearest: 1735959985961547789 - target: 1735959985851460218 at idx 31 +For topic gripper_width: timestamp difference is 111445156 for nearest: 1735959987528239456 - target: 1735959987416794300 at idx 46 +[INFO] [1735960010.189881]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit[pub_text-1] killing on exit +[INFO] [1735960010.939897]: Stored episode 1. +[INFO] [1735960010.940174]: Saving video... +[INFO] [1735960011.148081]: Video saved! +Starting episode 1 +[INFO] [1735960011.160429]: resetting environment +[INFO] [1735960011.164218]: cleared current plan +[INFO] [1735960011.164488]: moving to home +[INFO] [1735960014.966887]: reached home +[INFO] [1735960014.967505]: reset action history +[INFO] [1735960017.677400]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735960020.843628]: resetting environment +[INFO] [1735960020.849411]: cleared current plan +[INFO] [1735960020.849610]: moving to home +[INFO] [1735960021.851455]: reached home +[INFO] [1735960021.851871]: reset action history +[INFO] [1735960024.557726]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/episode_rosbags/episode_1_2025-01-03-22-07-04.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1978742.log +started roslaunch server http://158.130.50.37:45741/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1979161] +process[print_text-5]: started with pid [1979162] +process[rosbag_record-6]: started with pid [1979186] +started bagging! +[INFO] [1735960051.503044]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1735960052.232563]: Stored episode 2. +[INFO] [1735960052.234126]: Saving video... +[WARN] [1735960052.853691]: Plan exhausted +[WARN] [1735960052.890151]: Plan exhausted +[WARN] [1735960052.929802]: Plan exhausted +[INFO] [1735960052.919728]: Video saved! +Starting episode 2 +[INFO] [1735960052.953688]: resetting environment +[INFO] [1735960052.965267]: cleared current plan +[INFO] [1735960052.965532]: moving to home +[INFO] [1735960056.768655]: reached home +[INFO] [1735960056.770866]: reset action history +[INFO] [1735960059.504821]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735960066.856346]: resetting environment +[INFO] [1735960066.870435]: cleared current plan +[INFO] [1735960066.872229]: moving to home +[INFO] [1735960067.875224]: reached home +[INFO] [1735960067.877261]: reset action history +[INFO] [1735960070.586693]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/episode_rosbags/episode_2_2025-01-03-22-07-50.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1978742.log +started roslaunch server http://158.130.50.37:41191/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1979322] +process[print_text-8]: started with pid [1979323] +process[rosbag_record-9]: started with pid [1979324] +started bagging! +For topic gripper_width: timestamp difference is 101399089 for nearest: 1735960092528214122 - target: 1735960092426815033 at idx 249 +For topic gripper_width: timestamp difference is 100402812 for nearest: 1735960092728212098 - target: 1735960092627809286 at idx 249 +For topic gripper_width: timestamp difference is 100031487 for nearest: 1735960097328183524 - target: 1735960097228152037 at idx 249 +[INFO] [1735960097.732857]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit[pub_text-7] killing on exit +[INFO] [1735960098.493123]: Stored episode 3. +[INFO] [1735960098.500424]: Saving video... +[INFO] [1735960098.749693]: Video saved! +Starting episode 3 +[INFO] [1735960098.802600]: resetting environment +[INFO] [1735960098.820473]: cleared current plan +[INFO] [1735960098.825773]: moving to home +[INFO] [1735960102.336712]: reached home +[INFO] [1735960102.344855]: reset action history +[INFO] [1735960105.120596]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735960107.767876]: resetting environment +[INFO] [1735960107.810831]: cleared current plan +[INFO] [1735960107.814667]: moving to home +[INFO] [1735960108.817728]: reached home +[INFO] [1735960108.825277]: reset action history +[INFO] [1735960111.769687]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/episode_rosbags/episode_3_2025-01-03-22-08-31.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1978742.log +started roslaunch server http://158.130.50.37:37749/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1979433] +process[print_text-11]: started with pid [1979434] +process[rosbag_record-12]: started with pid [1979435] +started bagging! +For topic gripper_width: timestamp difference is -110497917 for nearest: 1735960125161547695 - target: 1735960125272045612 at idx 187 +For topic gripper_width: timestamp difference is -105175728 for nearest: 1735960131761547333 - target: 1735960131866723061 at idx 249 +For topic gripper_width: timestamp difference is -103245347 for nearest: 1735960134161557109 - target: 1735960134264802456 at idx 249 +[INFO] [1735960138.740608]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit[pub_text-10] killing on exit +[INFO] [1735960139.560185]: Stored episode 4. +[INFO] [1735960139.563634]: Saving video... +[INFO] [1735960140.005263]: Video saved! +Starting episode 4 +[INFO] [1735960140.047137]: resetting environment +[INFO] [1735960140.063349]: cleared current plan +[INFO] [1735960140.063600]: moving to home +[INFO] [1735960143.871680]: reached home +[INFO] [1735960143.874711]: reset action history +[INFO] [1735960146.667976]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735960150.298062]: resetting environment +[INFO] [1735960150.321684]: cleared current plan +[INFO] [1735960150.324406]: moving to home +[INFO] [1735960151.331827]: reached home +[INFO] [1735960151.335392]: reset action history +[INFO] [1735960154.108504]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/episode_rosbags/episode_4_2025-01-03-22-09-14.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1978742.log +started roslaunch server http://158.130.50.37:37943/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1979593] +process[print_text-14]: started with pid [1979594] +process[rosbag_record-15]: started with pid [1979595] +started bagging! +[INFO] [1735960182.462520]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[WARN] [1735960183.774573]: Plan exhausted +[WARN] [1735960183.818507]: Plan exhausted +[WARN] [1735960183.853071]: Plan exhausted +[WARN] [1735960183.893050]: Plan exhausted +[WARN] [1735960183.937934]: Plan exhausted +[WARN] [1735960183.971697]: Plan exhausted +[WARN] [1735960184.013220]: Plan exhausted +[INFO] [1735960183.247830]: Stored episode 5. +[INFO] [1735960183.256799]: Saving video... +[INFO] [1735960183.945842]: Video saved! +[INFO] [1735960183.996328]: resetting environment +[INFO] [1735960184.023424]: cleared current plan +[INFO] [1735960184.026399]: moving to home +[INFO] [1735960187.832913]: reached home +[INFO] [1735960187.838284]: reset action history +[INFO] [1735960190.655132]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/requirements.txt b/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/wandb-metadata.json b/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..db0fe63d1fd164d374d053c37c49fc918f4ca90f --- /dev/null +++ b/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-04T03:05:50.705915", + "startedAt": "2025-01-04T03:05:50.358113", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "c537eb6734155592809c2a7db5cca971ec186d72" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 4122.793375, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.025, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4497.434, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4496.663, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 818.3720664978027 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/wandb-summary.json b/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..b81d4e7a611a0f6fc8cc97a73814b8ae5ca9e526 --- /dev/null +++ b/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "4895a45f2099a6358d9b28958cd13550f10423ab4c8af10c7a84d1a6082182bb", "size": 1157651, "path": "media/videos/eval/0_eval_0_4895a45f2099a6358d9b.mp4"}, "global_step": 665, "_timestamp": 1735960194.0762923, "_runtime": 243.70739817619324, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "b53855a2640f91b72a281613afea099032a94909c5cce05b233d80567fe19579", "size": 1158998, "path": "media/videos/eval/1_eval_1_b53855a2640f91b72a28.mp4"}, "eval/num_success": 5.0, "episode": 15.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.4496806859970093, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.4799937605857849, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6638181209564209, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "0a282a3c65bafd15a5fd649e78be4615591940c6fa59a006eca48b8c88972e2d", "size": 1110522, "path": "media/videos/eval/2_eval_2_0a282a3c65bafd15a5fd.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "20a3393f5cad2132e7edf766a83eafcafb9865f2b2edf577a98a763bea1ebe2b", "size": 1138666, "path": "media/videos/eval/3_eval_3_20a3393f5cad2132e7ed.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "9f3ac500e502283fe4f2efdba56f7f6703259b0bb35f585c601ad045a4188d3f", "size": 1201784, "path": "media/videos/eval/4_eval_4_9f3ac500e502283fe4f2.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "1b420c32327aa0ba7f7504ab8676a845cd69d26134af4445f97766065563f5e1", "size": 313, "artifact_path": "wandb-client-artifact://cyuj2blcey0cususfojg8itbd4atiyld5wjkhwmy8g9spy208ck0cen4y0m26q5vzge6k2tpglnz858ky63zfgf3o6rguffk4vtxuhyix1sifehvbcwwznevwn1ol0im/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://3vh5ycjps4zptqtb15289jmk8i350t1ca4pu6gnzml2giwxljt6842pa605myccoo0ec5gf2fprbq1mvyurxvl6jhsl7k2ux9i5zsp1x26q5nkiq0q3hhxkm5xczu60m:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 14, "total_num_episodes": 15, "total_success_rate": 0.9333333333333333, "total_success_rate_lb_0.01": 0.6518780588799942, "total_success_rate_ub_0.01": 0.998813567583338, "total_success_rate_lb_0.025": 0.7327057420742016, "total_success_rate_ub_0.025": 0.9982055815802992, "total_success_rate_lb_0.05": 0.7706979363615327, "total_success_rate_ub_0.05": 0.9904621317001749, "total_success_rate_lb_0.1": 0.7770464345258705, "total_success_rate_ub_0.1": 0.9919731914013155, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "7fc509d15829a5499a649a124816119f58590cafd130d2d64d868db39049918b", "size": 519, "artifact_path": "wandb-client-artifact://0yqn70o7cytwufqz6zwxitztr13sivxqzp8xctkx7y9yvo5j19e2twxwl7tnqa4cs2pskwobl6pj2xrflhdxf0swlrbrsvblv2ba60b37i8ul0d0bti1kbrklq7way4p/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://cacbwtwfch44u8qzu46ycc24466qf97qki90b9hoc55lpxs4s0fiodwaudgslc9m1i88guy33nfok9qd1nxoa7r5cicl9mkac5lzp0x3ieascdn4q4b2xi92ns445wi0:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_7fc509d15829a5499a64.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 264}, "eval/total_success_rate": 0.9333333373069763, "eval/total_success_rate_lb_0.01": 0.6518780589103699, "eval/total_success_rate_ub_0.01": 0.9988135695457458, "eval/total_success_rate_lb_0.025": 0.7327057123184204, "eval/total_success_rate_ub_0.025": 0.9982056021690369, "eval/total_success_rate_lb_0.05": 0.7706979513168335, "eval/total_success_rate_ub_0.05": 0.9904621243476868, "eval/total_success_rate_lb_0.1": 0.7770464420318604, "eval/total_success_rate_ub_0.1": 0.9919731616973877} \ No newline at end of file diff --git a/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/logs/debug-internal.log b/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..465ebb441bb68d3b9cb714c3110d885c8378411b --- /dev/null +++ b/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/logs/debug-internal.log @@ -0,0 +1,774 @@ +2025-01-03 22:05:50,367 INFO StreamThr :1978913 [internal.py:wandb_internal():85] W&B internal server running at pid: 1978913, started at: 2025-01-03 22:05:50.366945 +2025-01-03 22:05:50,368 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status +2025-01-03 22:05:50,382 INFO WriterThread:1978913 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/run-pauhv4zk.wandb +2025-01-03 22:05:50,382 DEBUG SenderThread:1978913 [sender.py:send():379] send: header +2025-01-03 22:05:50,390 DEBUG SenderThread:1978913 [sender.py:send():379] send: run +2025-01-03 22:05:50,596 INFO SenderThread:1978913 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files +2025-01-03 22:05:50,596 INFO SenderThread:1978913 [sender.py:_start_run_threads():1188] run started: pauhv4zk with start time 1735959950.368894 +2025-01-03 22:05:50,612 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: check_version +2025-01-03 22:05:50,612 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: check_version +2025-01-03 22:05:50,687 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: run_start +2025-01-03 22:05:50,691 DEBUG HandlerThread:1978913 [system_info.py:__init__():26] System info init +2025-01-03 22:05:50,691 DEBUG HandlerThread:1978913 [system_info.py:__init__():41] System info init done +2025-01-03 22:05:50,692 INFO HandlerThread:1978913 [system_monitor.py:start():194] Starting system monitor +2025-01-03 22:05:50,692 INFO SystemMonitor:1978913 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-03 22:05:50,692 INFO HandlerThread:1978913 [system_monitor.py:probe():214] Collecting system info +2025-01-03 22:05:50,692 INFO SystemMonitor:1978913 [interfaces.py:start():188] Started cpu monitoring +2025-01-03 22:05:50,693 INFO SystemMonitor:1978913 [interfaces.py:start():188] Started disk monitoring +2025-01-03 22:05:50,693 INFO SystemMonitor:1978913 [interfaces.py:start():188] Started gpu monitoring +2025-01-03 22:05:50,694 INFO SystemMonitor:1978913 [interfaces.py:start():188] Started memory monitoring +2025-01-03 22:05:50,695 INFO SystemMonitor:1978913 [interfaces.py:start():188] Started network monitoring +2025-01-03 22:05:50,705 DEBUG HandlerThread:1978913 [system_info.py:probe():152] Probing system +2025-01-03 22:05:50,707 DEBUG HandlerThread:1978913 [system_info.py:_probe_git():137] Probing git +2025-01-03 22:05:50,711 DEBUG HandlerThread:1978913 [system_info.py:_probe_git():145] Probing git done +2025-01-03 22:05:50,711 DEBUG HandlerThread:1978913 [system_info.py:probe():200] Probing system done +2025-01-03 22:05:50,711 DEBUG HandlerThread:1978913 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-04T03:05:50.705915', 'startedAt': '2025-01-04T03:05:50.358113', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'c537eb6734155592809c2a7db5cca971ec186d72'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4122.793375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.025, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4497.434, 'min': 800.0, 'max': 4700.0}, {'current': 4496.663, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 818.3720664978027}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-03 22:05:50,712 INFO HandlerThread:1978913 [system_monitor.py:probe():224] Finished collecting system info +2025-01-03 22:05:50,712 INFO HandlerThread:1978913 [system_monitor.py:probe():227] Publishing system info +2025-01-03 22:05:50,712 DEBUG HandlerThread:1978913 [system_info.py:_save_code():44] Saving code +2025-01-03 22:05:50,713 DEBUG HandlerThread:1978913 [system_info.py:_save_code():67] Saving code done +2025-01-03 22:05:50,713 DEBUG HandlerThread:1978913 [system_info.py:_save_patches():84] Saving git patches +2025-01-03 22:05:50,779 DEBUG HandlerThread:1978913 [system_info.py:_save_patches():126] Saving git patches done +2025-01-03 22:05:50,779 INFO HandlerThread:1978913 [system_monitor.py:probe():229] Finished publishing system info +2025-01-03 22:05:50,782 DEBUG SenderThread:1978913 [sender.py:send():379] send: files +2025-01-03 22:05:50,782 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-03 22:05:50,782 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-03 22:05:50,783 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-03 22:05:51,016 INFO wandb-upload_2:1978913 [upload_job.py:push():130] Uploaded file /tmp/tmp3gj0zt0iwandb/88qa0844-diff.patch +2025-01-03 22:05:51,047 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: python_packages +2025-01-03 22:05:51,047 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: python_packages +2025-01-03 22:05:51,048 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:05:51,048 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:05:51,050 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:05:51,050 INFO HandlerThread:1978913 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/tb" + save: true +} +control { + relay_id: "559c5b11046641e18dc89c7d994a8133" +} +_info { + stream_id: "pauhv4zk" +} + +2025-01-03 22:05:51,050 DEBUG HandlerThread:1978913 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-03 22:05:51,078 INFO wandb-upload_0:1978913 [upload_job.py:push():130] Uploaded file /tmp/tmp3gj0zt0iwandb/h3klqk8s-wandb-metadata.json +2025-01-03 22:05:51,097 INFO wandb-upload_1:1978913 [upload_job.py:push():130] Uploaded file /tmp/tmp3gj0zt0iwandb/2nlk63jw-code/FISH/eval_robot.py +2025-01-03 22:05:51,150 DEBUG SenderThread:1978913 [sender.py:send():379] send: telemetry +2025-01-03 22:05:51,151 DEBUG SenderThread:1978913 [sender.py:send():379] send: tbrecord +2025-01-03 22:05:51,151 DEBUG SenderThread:1978913 [sender.py:send():379] send: files +2025-01-03 22:05:51,151 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file events.out.tfevents.1735959951.leonmkim-ROG-Strix-G15CS-G15CS.1978742.0 with policy live +2025-01-03 22:05:51,336 INFO wandb-upload_2:1978913 [upload_job.py:push():130] Uploaded file /tmp/tmp3gj0zt0iwandb/zqz83b5d-events.out.tfevents.1735959951.leonmkim-ROG-Strix-G15CS-G15CS.1978742.0 +2025-01-03 22:05:51,598 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/requirements.txt +2025-01-03 22:05:51,598 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:05:51,598 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/wandb-metadata.json +2025-01-03 22:05:51,598 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/code/FISH/eval_robot.py +2025-01-03 22:05:51,598 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/diff.patch +2025-01-03 22:05:51,598 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/events.out.tfevents.1735959951.leonmkim-ROG-Strix-G15CS-G15CS.1978742.0 +2025-01-03 22:05:51,598 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/code +2025-01-03 22:05:51,598 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/code/FISH +2025-01-03 22:05:52,048 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:05:53,048 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:05:53,597 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:05:54,049 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:05:55,049 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:05:56,048 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:05:56,049 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:05:57,049 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:05:58,049 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:05:59,049 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:00,049 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:01,048 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:06:01,049 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:02,050 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:03,050 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:04,050 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:05,050 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:06,048 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:06:06,048 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:06:06,089 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:06,196 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:06:07,050 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:08,050 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:09,051 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:09,317 DEBUG SenderThread:1978913 [sender.py:send():379] send: config +2025-01-03 22:06:10,051 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:11,051 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:11,602 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:06:12,049 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:06:12,051 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:13,077 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:14,085 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:15,052 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:15,603 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:06:16,052 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:17,052 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:17,797 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:06:18,052 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:19,052 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:19,604 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:06:20,052 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:21,048 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:06:21,048 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:06:21,052 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:21,605 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:06:22,052 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:23,053 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:23,112 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:06:23,605 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/config.yaml +2025-01-03 22:06:24,053 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:25,053 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:25,606 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:06:26,053 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:27,053 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:27,606 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:06:28,053 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:28,581 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:06:29,057 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:29,607 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:06:30,053 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:31,053 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:32,053 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:33,053 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:34,051 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:06:34,053 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:35,054 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:36,048 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:06:36,048 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:06:36,089 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:37,054 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:38,054 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:39,054 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:39,123 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:06:40,054 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:41,054 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:42,055 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:43,055 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:44,055 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:45,050 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:06:45,055 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:46,055 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:47,055 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:48,055 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:49,055 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:50,056 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:50,190 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:06:50,695 DEBUG SystemMonitor:1978913 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-03 22:06:50,696 DEBUG SenderThread:1978913 [sender.py:send():379] send: stats +2025-01-03 22:06:51,050 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:06:51,051 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:06:51,057 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:51,193 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:06:51,194 DEBUG SenderThread:1978913 [sender.py:send():379] send: files +2025-01-03 22:06:51,194 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_4895a45f2099a6358d9b.mp4 with policy now +2025-01-03 22:06:51,194 DEBUG SenderThread:1978913 [sender.py:send():379] send: history +2025-01-03 22:06:51,194 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:06:51,195 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:06:51,473 INFO wandb-upload_1:1978913 [upload_job.py:push():130] Uploaded file /tmp/tmp3gj0zt0iwandb/xox0l4vk-media/videos/eval/0_eval_0_4895a45f2099a6358d9b.mp4 +2025-01-03 22:06:51,612 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:06:51,613 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval/0_eval_0_4895a45f2099a6358d9b.mp4 +2025-01-03 22:06:51,613 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/wandb-summary.json +2025-01-03 22:06:51,613 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval +2025-01-03 22:06:51,613 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media +2025-01-03 22:06:51,613 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos +2025-01-03 22:06:51,873 INFO wandb-upload_2:1978913 [upload_job.py:push():130] Uploaded file /tmp/tmp3gj0zt0iwandb/yf0j5h9j-media/videos/eval/0_eval_0_4895a45f2099a6358d9b.mp4 +2025-01-03 22:06:52,057 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:53,057 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:53,613 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:06:54,057 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:55,060 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:56,009 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:06:56,058 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:57,060 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:57,614 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:06:58,060 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:59,060 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:06:59,615 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:07:00,060 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:01,052 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:07:01,060 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:01,616 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/events.out.tfevents.1735959951.leonmkim-ROG-Strix-G15CS-G15CS.1978742.0 +2025-01-03 22:07:01,904 INFO wandb-upload_1:1978913 [upload_job.py:push():130] Uploaded file /tmp/tmp3gj0zt0iwandb/h81fxdeo-events.out.tfevents.1735959951.leonmkim-ROG-Strix-G15CS-G15CS.1978742.0 +2025-01-03 22:07:02,060 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:03,061 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:03,617 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:07:04,061 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:05,061 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:05,617 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:07:06,049 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:07:06,049 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:07:06,089 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:06,206 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:07:07,061 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:07,618 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:07:08,061 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:09,061 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:10,064 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:11,061 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:12,052 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:07:12,063 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:13,062 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:14,062 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:15,062 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:16,062 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:17,052 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:07:17,062 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:18,062 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:19,062 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:20,062 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:20,696 DEBUG SenderThread:1978913 [sender.py:send():379] send: stats +2025-01-03 22:07:21,049 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:07:21,049 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:07:21,097 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:22,063 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:22,208 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:07:23,063 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:24,063 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:25,063 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:26,063 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:27,063 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:28,052 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:07:28,063 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:29,064 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:30,064 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:31,064 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:32,064 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:32,950 DEBUG SenderThread:1978913 [sender.py:send():379] send: files +2025-01-03 22:07:32,950 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_b53855a2640f91b72a28.mp4 with policy now +2025-01-03 22:07:32,951 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:07:32,954 DEBUG SenderThread:1978913 [sender.py:send():379] send: history +2025-01-03 22:07:32,954 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:07:32,954 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:07:33,064 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:33,217 INFO wandb-upload_2:1978913 [upload_job.py:push():130] Uploaded file /tmp/tmp3gj0zt0iwandb/edye93xv-media/videos/eval/1_eval_1_b53855a2640f91b72a28.mp4 +2025-01-03 22:07:33,626 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/wandb-summary.json +2025-01-03 22:07:33,626 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:07:33,626 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval/1_eval_1_b53855a2640f91b72a28.mp4 +2025-01-03 22:07:33,627 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval +2025-01-03 22:07:33,919 INFO wandb-upload_1:1978913 [upload_job.py:push():130] Uploaded file /tmp/tmp3gj0zt0iwandb/c1bote73-media/videos/eval/1_eval_1_b53855a2640f91b72a28.mp4 +2025-01-03 22:07:33,966 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:07:34,069 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:35,081 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:35,627 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:07:36,054 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:07:36,054 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:07:36,097 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:37,081 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:37,627 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:07:38,077 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:39,080 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:39,506 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:07:39,628 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:07:40,075 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:41,075 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:41,629 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:07:42,076 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:43,076 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:44,076 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:45,055 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:07:45,076 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:46,076 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:47,076 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:47,473 DEBUG SenderThread:1978913 [sender.py:send():379] send: history +2025-01-03 22:07:47,473 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:07:47,473 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:07:47,631 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/wandb-summary.json +2025-01-03 22:07:47,631 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/events.out.tfevents.1735959951.leonmkim-ROG-Strix-G15CS-G15CS.1978742.0 +2025-01-03 22:07:48,078 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:49,077 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:49,632 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:07:50,077 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:50,587 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:07:50,697 DEBUG SenderThread:1978913 [sender.py:send():379] send: stats +2025-01-03 22:07:51,051 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:07:51,052 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:07:51,077 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:51,632 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:07:52,077 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:53,077 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:53,633 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:07:54,077 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:55,077 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:56,057 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:07:56,077 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:57,077 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:58,077 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:07:59,081 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:00,079 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:01,058 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:08:01,079 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:02,080 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:03,080 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:04,086 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:05,086 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:06,052 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:08:06,052 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:08:06,093 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:06,135 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:08:07,098 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:08,086 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:09,086 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:10,087 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:11,087 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:12,058 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:08:12,087 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:13,087 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:13,639 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:08:14,087 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:15,087 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:16,092 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:17,058 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:08:17,101 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:18,103 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:18,788 DEBUG SenderThread:1978913 [sender.py:send():379] send: files +2025-01-03 22:08:18,788 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_0a282a3c65bafd15a5fd.mp4 with policy now +2025-01-03 22:08:18,796 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:08:18,797 DEBUG SenderThread:1978913 [sender.py:send():379] send: history +2025-01-03 22:08:18,797 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:08:18,797 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:08:19,050 INFO wandb-upload_2:1978913 [upload_job.py:push():130] Uploaded file /tmp/tmp3gj0zt0iwandb/2bkqasp9-media/videos/eval/2_eval_2_0a282a3c65bafd15a5fd.mp4 +2025-01-03 22:08:19,108 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:19,641 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/wandb-summary.json +2025-01-03 22:08:19,641 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:08:19,641 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval/2_eval_2_0a282a3c65bafd15a5fd.mp4 +2025-01-03 22:08:19,642 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval +2025-01-03 22:08:19,915 INFO wandb-upload_1:1978913 [upload_job.py:push():130] Uploaded file /tmp/tmp3gj0zt0iwandb/lhfrs9d4-media/videos/eval/2_eval_2_0a282a3c65bafd15a5fd.mp4 +2025-01-03 22:08:20,106 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:20,697 DEBUG SenderThread:1978913 [sender.py:send():379] send: stats +2025-01-03 22:08:21,057 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:08:21,058 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:08:21,110 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:21,642 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:08:22,111 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:22,150 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:08:23,108 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:23,643 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:08:24,112 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:25,113 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:26,115 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:27,114 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:27,644 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:08:27,747 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:08:28,115 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:28,303 DEBUG SenderThread:1978913 [sender.py:send():379] send: history +2025-01-03 22:08:28,303 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:08:28,309 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:08:28,644 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/wandb-summary.json +2025-01-03 22:08:28,644 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/events.out.tfevents.1735959951.leonmkim-ROG-Strix-G15CS-G15CS.1978742.0 +2025-01-03 22:08:28,888 INFO wandb-upload_2:1978913 [upload_job.py:push():130] Uploaded file /tmp/tmp3gj0zt0iwandb/yghzxc38-events.out.tfevents.1735959951.leonmkim-ROG-Strix-G15CS-G15CS.1978742.0 +2025-01-03 22:08:29,119 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:29,645 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:08:30,122 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:31,132 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:31,645 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:08:32,119 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:33,019 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:08:33,119 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:33,646 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:08:34,119 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:35,119 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:36,053 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:08:36,054 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:08:36,119 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:37,120 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:38,120 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:38,213 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:08:39,120 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:40,120 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:41,120 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:42,120 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:43,120 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:44,069 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:08:44,120 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:45,121 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:46,120 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:47,121 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:47,651 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:08:48,121 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:49,069 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:08:49,121 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:50,121 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:50,698 DEBUG SenderThread:1978913 [sender.py:send():379] send: stats +2025-01-03 22:08:51,054 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:08:51,054 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:08:51,121 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:52,122 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:53,122 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:53,653 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:08:54,122 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:54,351 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:08:55,122 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:55,654 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:08:56,122 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:57,122 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:58,122 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:59,124 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:08:59,561 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:08:59,655 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:09:00,039 DEBUG SenderThread:1978913 [sender.py:send():379] send: files +2025-01-03 22:09:00,039 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_20a3393f5cad2132e7ed.mp4 with policy now +2025-01-03 22:09:00,041 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:09:00,042 DEBUG SenderThread:1978913 [sender.py:send():379] send: history +2025-01-03 22:09:00,042 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:09:00,042 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:09:00,124 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:00,307 INFO wandb-upload_1:1978913 [upload_job.py:push():130] Uploaded file /tmp/tmp3gj0zt0iwandb/687zeufw-media/videos/eval/3_eval_3_20a3393f5cad2132e7ed.mp4 +2025-01-03 22:09:00,655 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/wandb-summary.json +2025-01-03 22:09:00,655 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval/3_eval_3_20a3393f5cad2132e7ed.mp4 +2025-01-03 22:09:00,656 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval +2025-01-03 22:09:00,948 INFO wandb-upload_2:1978913 [upload_job.py:push():130] Uploaded file /tmp/tmp3gj0zt0iwandb/fw8qj3ez-media/videos/eval/3_eval_3_20a3393f5cad2132e7ed.mp4 +2025-01-03 22:09:01,137 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:01,656 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:09:02,138 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:03,133 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:04,140 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:04,917 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:09:05,139 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:05,657 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:09:06,057 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:09:06,057 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:09:06,141 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:07,132 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:07,658 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:09:08,136 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:09,137 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:10,136 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:10,275 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:09:10,358 DEBUG SenderThread:1978913 [sender.py:send():379] send: history +2025-01-03 22:09:10,358 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:09:10,359 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:09:10,659 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/wandb-summary.json +2025-01-03 22:09:10,659 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/events.out.tfevents.1735959951.leonmkim-ROG-Strix-G15CS-G15CS.1978742.0 +2025-01-03 22:09:11,136 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:11,659 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:09:12,150 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:13,144 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:13,660 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:09:14,153 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:15,141 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:15,391 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:09:15,660 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:09:16,139 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:17,139 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:17,661 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:09:18,139 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:19,139 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:20,139 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:20,698 DEBUG SenderThread:1978913 [sender.py:send():379] send: stats +2025-01-03 22:09:20,700 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:09:21,055 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:09:21,055 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:09:21,139 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:22,139 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:23,139 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:24,140 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:25,140 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:26,072 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:09:26,140 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:27,140 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:28,140 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:29,140 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:30,140 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:31,072 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:09:31,141 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:32,141 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:33,141 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:34,141 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:35,141 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:36,055 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:09:36,055 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:09:36,141 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:36,207 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:09:37,141 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:38,141 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:39,141 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:40,142 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:41,142 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:42,072 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:09:42,142 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:43,145 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:43,670 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:09:43,991 DEBUG SenderThread:1978913 [sender.py:send():379] send: files +2025-01-03 22:09:43,991 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_9f3ac500e502283fe4f2.mp4 with policy now +2025-01-03 22:09:43,995 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:09:43,996 DEBUG SenderThread:1978913 [sender.py:send():379] send: history +2025-01-03 22:09:43,996 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:09:43,996 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:09:44,150 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:44,251 INFO wandb-upload_1:1978913 [upload_job.py:push():130] Uploaded file /tmp/tmp3gj0zt0iwandb/fqs0z2x1-media/videos/eval/4_eval_4_9f3ac500e502283fe4f2.mp4 +2025-01-03 22:09:44,670 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/wandb-summary.json +2025-01-03 22:09:44,670 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval/4_eval_4_9f3ac500e502283fe4f2.mp4 +2025-01-03 22:09:44,671 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval +2025-01-03 22:09:44,977 INFO wandb-upload_2:1978913 [upload_job.py:push():130] Uploaded file /tmp/tmp3gj0zt0iwandb/syesmre4-media/videos/eval/4_eval_4_9f3ac500e502283fe4f2.mp4 +2025-01-03 22:09:45,153 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:45,671 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:09:46,150 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:47,088 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:09:47,158 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:48,159 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:49,156 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:49,672 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:09:50,152 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:50,698 DEBUG SenderThread:1978913 [sender.py:send():379] send: stats +2025-01-03 22:09:51,064 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:09:51,071 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:09:51,157 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:51,673 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:09:52,164 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:52,369 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:09:52,673 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/events.out.tfevents.1735959951.leonmkim-ROG-Strix-G15CS-G15CS.1978742.0 +2025-01-03 22:09:52,936 INFO wandb-upload_1:1978913 [upload_job.py:push():130] Uploaded file /tmp/tmp3gj0zt0iwandb/p08qcqy3-events.out.tfevents.1735959951.leonmkim-ROG-Strix-G15CS-G15CS.1978742.0 +2025-01-03 22:09:53,156 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:53,412 DEBUG SenderThread:1978913 [sender.py:send():379] send: history +2025-01-03 22:09:53,412 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:09:53,413 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:09:53,609 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 22:09:53,609 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 22:09:53,673 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:09:53,674 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/wandb-summary.json +2025-01-03 22:09:53,674 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/events.out.tfevents.1735959951.leonmkim-ROG-Strix-G15CS-G15CS.1978742.0 +2025-01-03 22:09:53,674 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 22:09:53,674 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media +2025-01-03 22:09:53,674 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/table/eval +2025-01-03 22:09:53,674 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/table +2025-01-03 22:09:53,802 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:09:53,854 INFO wandb-upload_2:1978913 [upload_job.py:push():130] Uploaded file /tmp/tmp3gj0zt0iwandb/ml3llhfk-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 22:09:54,156 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:54,159 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 22:09:54,201 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 22:09:54,222 INFO wandb-upload_1:1978913 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpjt3eh8h1 +2025-01-03 22:09:54,592 INFO SenderThread:1978913 [sender.py:send_request_log_artifact():1525] logged artifact run-pauhv4zk-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzk1NTM4MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY4NDQ1OQ==', 'latestArtifact': None}} +2025-01-03 22:09:54,592 DEBUG SenderThread:1978913 [sender.py:send():379] send: files +2025-01-03 22:09:54,592 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-03 22:09:54,592 DEBUG SenderThread:1978913 [sender.py:send():379] send: history +2025-01-03 22:09:54,593 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:09:54,593 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:09:54,593 DEBUG SenderThread:1978913 [sender.py:send():379] send: telemetry +2025-01-03 22:09:54,593 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:09:54,597 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:09:54,598 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:09:54,598 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:09:54,598 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:09:54,598 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:09:54,599 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:09:54,599 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:09:54,599 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:09:54,599 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:09:54,600 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:09:54,600 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:09:54,600 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:09:54,600 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:09:54,601 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:09:54,601 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:09:54,601 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:09:54,601 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:09:54,601 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:09:54,602 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:09:54,602 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:09:54,602 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:09:54,602 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 22:09:54,674 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/wandb-summary.json +2025-01-03 22:09:54,674 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/events.out.tfevents.1735959951.leonmkim-ROG-Strix-G15CS-G15CS.1978742.0 +2025-01-03 22:09:54,674 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/table/eval/total_success_rate_ci_6_7fc509d15829a5499a64.table.json +2025-01-03 22:09:54,674 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/table/eval +2025-01-03 22:09:54,828 INFO wandb-upload_2:1978913 [upload_job.py:push():130] Uploaded file /tmp/tmp3gj0zt0iwandb/awjizvt9-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 22:09:54,891 INFO wandb-upload_1:1978913 [upload_job.py:push():130] Uploaded file /tmp/tmp3gj0zt0iwandb/qkz34qym-media/table/eval/total_success_rate_ci_6_7fc509d15829a5499a64.table.json +2025-01-03 22:09:55,001 INFO wandb-upload_0:1978913 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpt85cl7oz +2025-01-03 22:09:55,156 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:55,365 INFO SenderThread:1978913 [sender.py:send_request_log_artifact():1525] logged artifact run-pauhv4zk-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzk1NTM5Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY4NDQ2Mg==', 'latestArtifact': None}} +2025-01-03 22:09:55,365 DEBUG SenderThread:1978913 [sender.py:send():379] send: files +2025-01-03 22:09:55,365 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_7fc509d15829a5499a64.table.json with policy now +2025-01-03 22:09:55,365 DEBUG SenderThread:1978913 [sender.py:send():379] send: history +2025-01-03 22:09:55,366 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:09:55,366 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:09:55,630 INFO wandb-upload_2:1978913 [upload_job.py:push():130] Uploaded file /tmp/tmp3gj0zt0iwandb/gltmt24r-media/table/eval/total_success_rate_ci_6_7fc509d15829a5499a64.table.json +2025-01-03 22:09:55,674 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/wandb-summary.json +2025-01-03 22:09:55,674 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:09:56,156 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:57,157 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:58,077 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:09:58,157 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:09:59,157 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:00,157 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:01,157 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:02,157 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:03,157 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:04,077 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:10:04,157 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:05,158 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:06,055 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 22:10:06,055 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 22:10:06,158 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:07,158 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:08,158 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:09,123 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:10:09,158 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:09,680 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/config.yaml +2025-01-03 22:10:10,158 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:11,159 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:12,159 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:13,159 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:14,159 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:15,025 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:10:15,069 DEBUG SenderThread:1978913 [sender.py:send():379] send: exit +2025-01-03 22:10:15,070 INFO SenderThread:1978913 [sender.py:send_exit():586] handling exit code: 0 +2025-01-03 22:10:15,070 INFO SenderThread:1978913 [sender.py:send_exit():588] handling runtime: 264 +2025-01-03 22:10:15,070 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:10:15,070 INFO SenderThread:1978913 [sender.py:send_exit():594] send defer +2025-01-03 22:10:15,070 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:10:15,070 INFO HandlerThread:1978913 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-03 22:10:15,070 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: defer +2025-01-03 22:10:15,070 INFO SenderThread:1978913 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-03 22:10:15,071 INFO SenderThread:1978913 [sender.py:transition_state():614] send defer: 1 +2025-01-03 22:10:15,071 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:10:15,071 INFO HandlerThread:1978913 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-03 22:10:15,071 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: defer +2025-01-03 22:10:15,071 INFO SenderThread:1978913 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-03 22:10:15,071 INFO SenderThread:1978913 [sender.py:transition_state():614] send defer: 2 +2025-01-03 22:10:15,071 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:10:15,071 INFO HandlerThread:1978913 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-03 22:10:15,071 INFO HandlerThread:1978913 [system_monitor.py:finish():203] Stopping system monitor +2025-01-03 22:10:15,071 DEBUG SystemMonitor:1978913 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-03 22:10:15,071 DEBUG SystemMonitor:1978913 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-03 22:10:15,072 INFO HandlerThread:1978913 [interfaces.py:finish():200] Joined cpu monitor +2025-01-03 22:10:15,072 INFO HandlerThread:1978913 [interfaces.py:finish():200] Joined disk monitor +2025-01-03 22:10:15,125 INFO HandlerThread:1978913 [interfaces.py:finish():200] Joined gpu monitor +2025-01-03 22:10:15,125 INFO HandlerThread:1978913 [interfaces.py:finish():200] Joined memory monitor +2025-01-03 22:10:15,126 INFO HandlerThread:1978913 [interfaces.py:finish():200] Joined network monitor +2025-01-03 22:10:15,126 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: defer +2025-01-03 22:10:15,126 INFO SenderThread:1978913 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-03 22:10:15,126 INFO SenderThread:1978913 [sender.py:transition_state():614] send defer: 3 +2025-01-03 22:10:15,126 DEBUG SenderThread:1978913 [sender.py:send():379] send: stats +2025-01-03 22:10:15,126 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:10:15,126 INFO HandlerThread:1978913 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-03 22:10:15,127 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: defer +2025-01-03 22:10:15,127 INFO SenderThread:1978913 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-03 22:10:15,127 INFO SenderThread:1978913 [sender.py:transition_state():614] send defer: 4 +2025-01-03 22:10:15,127 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:10:15,127 INFO HandlerThread:1978913 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-03 22:10:15,682 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/wandb-summary.json +2025-01-03 22:10:17,683 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:10:21,419 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:10:21,419 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:10:21,419 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 22:10:21,421 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: defer +2025-01-03 22:10:21,421 INFO SenderThread:1978913 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-03 22:10:21,421 INFO SenderThread:1978913 [sender.py:transition_state():614] send defer: 5 +2025-01-03 22:10:21,421 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:10:21,421 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:10:21,421 INFO HandlerThread:1978913 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-03 22:10:21,422 DEBUG SenderThread:1978913 [sender.py:send():379] send: history +2025-01-03 22:10:21,422 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 22:10:21,422 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:10:21,423 DEBUG SenderThread:1978913 [sender.py:send():379] send: summary +2025-01-03 22:10:21,427 INFO SenderThread:1978913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 22:10:21,427 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: defer +2025-01-03 22:10:21,427 INFO SenderThread:1978913 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-03 22:10:21,427 INFO SenderThread:1978913 [sender.py:transition_state():614] send defer: 6 +2025-01-03 22:10:21,427 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:10:21,427 INFO HandlerThread:1978913 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-03 22:10:21,427 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: defer +2025-01-03 22:10:21,427 INFO SenderThread:1978913 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-03 22:10:21,427 INFO SenderThread:1978913 [sender.py:transition_state():614] send defer: 7 +2025-01-03 22:10:21,427 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 22:10:21,428 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:10:21,428 INFO HandlerThread:1978913 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-03 22:10:21,428 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: defer +2025-01-03 22:10:21,428 INFO SenderThread:1978913 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-03 22:10:21,685 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/wandb-summary.json +2025-01-03 22:10:22,071 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:10:23,033 INFO SenderThread:1978913 [sender.py:transition_state():614] send defer: 8 +2025-01-03 22:10:23,033 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:10:23,033 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:10:23,034 INFO HandlerThread:1978913 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-03 22:10:23,034 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: defer +2025-01-03 22:10:23,034 INFO SenderThread:1978913 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-03 22:10:23,034 INFO SenderThread:1978913 [job_builder.py:build():440] Attempting to build job artifact +2025-01-03 22:10:23,036 INFO SenderThread:1978913 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-03 22:10:23,038 INFO SenderThread:1978913 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-03 22:10:23,048 INFO SenderThread:1978913 [sender.py:transition_state():614] send defer: 9 +2025-01-03 22:10:23,048 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:10:23,048 INFO HandlerThread:1978913 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-03 22:10:23,048 DEBUG SenderThread:1978913 [sender.py:send():379] send: artifact +2025-01-03 22:10:23,071 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:10:23,397 INFO wandb-upload_1:1978913 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpjj1azryb +2025-01-03 22:10:23,397 INFO wandb-upload_2:1978913 [upload_job.py:push():85] Skipped uploading /tmp/tmp31jrvtyv/wandb-job.json +2025-01-03 22:10:23,507 INFO wandb-upload_0:1978913 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpn9gg5cm_ +2025-01-03 22:10:23,685 INFO Thread-12 :1978913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:10:24,100 INFO SenderThread:1978913 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNzk1NTgzMA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNzkwNDU3OA==', 'versionIndex': 410}}} +2025-01-03 22:10:24,100 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: defer +2025-01-03 22:10:24,100 INFO SenderThread:1978913 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-03 22:10:24,100 INFO SenderThread:1978913 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-03 22:10:24,686 INFO SenderThread:1978913 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files +2025-01-03 22:10:24,686 INFO SenderThread:1978913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/config.yaml config.yaml +2025-01-03 22:10:24,686 INFO SenderThread:1978913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/wandb-summary.json wandb-summary.json +2025-01-03 22:10:24,687 INFO SenderThread:1978913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/events.out.tfevents.1735959951.leonmkim-ROG-Strix-G15CS-G15CS.1978742.0 events.out.tfevents.1735959951.leonmkim-ROG-Strix-G15CS-G15CS.1978742.0 +2025-01-03 22:10:24,687 INFO SenderThread:1978913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/requirements.txt requirements.txt +2025-01-03 22:10:24,687 INFO SenderThread:1978913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/wandb-metadata.json wandb-metadata.json +2025-01-03 22:10:24,687 INFO SenderThread:1978913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log output.log +2025-01-03 22:10:24,687 INFO SenderThread:1978913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/diff.patch diff.patch +2025-01-03 22:10:24,691 INFO SenderThread:1978913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-03 22:10:24,691 INFO SenderThread:1978913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval/0_eval_0_4895a45f2099a6358d9b.mp4 media/videos/eval/0_eval_0_4895a45f2099a6358d9b.mp4 +2025-01-03 22:10:24,691 INFO SenderThread:1978913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval/1_eval_1_b53855a2640f91b72a28.mp4 media/videos/eval/1_eval_1_b53855a2640f91b72a28.mp4 +2025-01-03 22:10:24,691 INFO SenderThread:1978913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval/4_eval_4_9f3ac500e502283fe4f2.mp4 media/videos/eval/4_eval_4_9f3ac500e502283fe4f2.mp4 +2025-01-03 22:10:24,691 INFO SenderThread:1978913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval/3_eval_3_20a3393f5cad2132e7ed.mp4 media/videos/eval/3_eval_3_20a3393f5cad2132e7ed.mp4 +2025-01-03 22:10:24,691 INFO SenderThread:1978913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/videos/eval/2_eval_2_0a282a3c65bafd15a5fd.mp4 media/videos/eval/2_eval_2_0a282a3c65bafd15a5fd.mp4 +2025-01-03 22:10:24,693 INFO SenderThread:1978913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/table/eval/total_success_rate_ci_6_7fc509d15829a5499a64.table.json media/table/eval/total_success_rate_ci_6_7fc509d15829a5499a64.table.json +2025-01-03 22:10:24,693 INFO SenderThread:1978913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 22:10:24,693 INFO SenderThread:1978913 [sender.py:transition_state():614] send defer: 10 +2025-01-03 22:10:24,693 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:10:24,696 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:10:24,696 INFO HandlerThread:1978913 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-03 22:10:24,696 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: defer +2025-01-03 22:10:24,696 INFO SenderThread:1978913 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-03 22:10:24,696 INFO SenderThread:1978913 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 22:10:24,878 INFO wandb-upload_1:1978913 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/config.yaml +2025-01-03 22:10:24,891 INFO wandb-upload_4:1978913 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/requirements.txt +2025-01-03 22:10:24,896 INFO wandb-upload_2:1978913 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/wandb-summary.json +2025-01-03 22:10:24,929 INFO wandb-upload_0:1978913 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/files/output.log +2025-01-03 22:10:24,932 INFO wandb-upload_3:1978913 [upload_job.py:push():130] Uploaded file /tmp/tmp3gj0zt0iwandb/6nvkzf8o-events.out.tfevents.1735959951.leonmkim-ROG-Strix-G15CS-G15CS.1978742.0 +2025-01-03 22:10:25,072 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:10:25,072 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:10:25,133 INFO Thread-11 (_thread_body):1978913 [sender.py:transition_state():614] send defer: 11 +2025-01-03 22:10:25,133 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:10:25,133 INFO HandlerThread:1978913 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-03 22:10:25,133 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: defer +2025-01-03 22:10:25,133 INFO SenderThread:1978913 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-03 22:10:25,133 INFO SenderThread:1978913 [file_pusher.py:join():175] waiting for file pusher +2025-01-03 22:10:25,135 INFO SenderThread:1978913 [sender.py:transition_state():614] send defer: 12 +2025-01-03 22:10:25,135 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:10:25,135 INFO HandlerThread:1978913 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-03 22:10:25,135 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: defer +2025-01-03 22:10:25,135 INFO SenderThread:1978913 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-03 22:10:25,135 INFO SenderThread:1978913 [file_stream.py:finish():601] file stream finish called +2025-01-03 22:10:25,334 INFO SenderThread:1978913 [file_stream.py:finish():605] file stream finish is done +2025-01-03 22:10:25,334 INFO SenderThread:1978913 [sender.py:transition_state():614] send defer: 13 +2025-01-03 22:10:25,334 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:10:25,334 INFO HandlerThread:1978913 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-03 22:10:25,334 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: defer +2025-01-03 22:10:25,335 INFO SenderThread:1978913 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-03 22:10:25,335 INFO SenderThread:1978913 [sender.py:transition_state():614] send defer: 14 +2025-01-03 22:10:25,335 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 22:10:25,335 INFO HandlerThread:1978913 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-03 22:10:25,335 DEBUG SenderThread:1978913 [sender.py:send():379] send: final +2025-01-03 22:10:25,335 DEBUG SenderThread:1978913 [sender.py:send():379] send: footer +2025-01-03 22:10:25,335 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: defer +2025-01-03 22:10:25,335 INFO SenderThread:1978913 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-03 22:10:25,336 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:10:25,336 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:10:25,336 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: server_info +2025-01-03 22:10:25,336 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 22:10:25,336 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: get_summary +2025-01-03 22:10:25,337 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-03 22:10:25,337 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: server_info +2025-01-03 22:10:25,339 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 22:10:25,384 DEBUG SenderThread:1978913 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 22:10:25,384 INFO MainThread:1978913 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-03 22:10:25,385 INFO MainThread:1978913 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-03 22:10:25,385 INFO MainThread:1978913 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-03 22:10:25,385 DEBUG HandlerThread:1978913 [handler.py:handle_request():158] handle_request: shutdown +2025-01-03 22:10:25,385 INFO HandlerThread:1978913 [handler.py:finish():882] shutting down handler +2025-01-03 22:10:26,337 INFO WriterThread:1978913 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/run-pauhv4zk.wandb +2025-01-03 22:10:26,384 INFO SenderThread:1978913 [sender.py:finish():1615] shutting down sender +2025-01-03 22:10:26,385 INFO SenderThread:1978913 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 22:10:26,385 INFO SenderThread:1978913 [file_pusher.py:join():175] waiting for file pusher diff --git a/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/logs/debug.log b/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..3fa42ae73507a62cffec551eae9e8e988be7f946 --- /dev/null +++ b/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-03 22:05:50,359 INFO MainThread:1978742 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-03 22:05:50,359 INFO MainThread:1978742 [wandb_setup.py:_flush():76] Configure stats pid to 1978742 +2025-01-03 22:05:50,359 INFO MainThread:1978742 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-03 22:05:50,359 INFO MainThread:1978742 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/settings +2025-01-03 22:05:50,359 INFO MainThread:1978742 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-03 22:05:50,359 INFO MainThread:1978742 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-03 22:05:50,359 INFO MainThread:1978742 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-03 22:05:50,359 INFO MainThread:1978742 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-03 22:05:50,359 INFO MainThread:1978742 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/logs/debug.log +2025-01-03 22:05:50,359 INFO MainThread:1978742 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/logs/debug-internal.log +2025-01-03 22:05:50,359 INFO MainThread:1978742 [wandb_init.py:init():569] calling init triggers +2025-01-03 22:05:50,360 INFO MainThread:1978742 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1938_0', 'wandb_notes': '1938_0_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk'} +2025-01-03 22:05:50,360 INFO MainThread:1978742 [wandb_init.py:init():619] starting backend +2025-01-03 22:05:50,361 INFO MainThread:1978742 [wandb_init.py:init():623] setting up manager +2025-01-03 22:05:50,366 INFO MainThread:1978742 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-03 22:05:50,368 INFO MainThread:1978742 [wandb_init.py:init():631] backend started and connected +2025-01-03 22:05:50,381 INFO MainThread:1978742 [wandb_init.py:init():720] updated telemetry +2025-01-03 22:05:50,388 INFO MainThread:1978742 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-03 22:05:50,612 INFO MainThread:1978742 [wandb_run.py:_on_init():2435] communicating current version +2025-01-03 22:05:50,682 INFO MainThread:1978742 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-03 22:05:50,682 INFO MainThread:1978742 [wandb_init.py:init():804] starting run threads in backend +2025-01-03 22:05:51,048 INFO MainThread:1978742 [wandb_run.py:_console_start():2413] atexit reg +2025-01-03 22:05:51,048 INFO MainThread:1978742 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-03 22:05:51,048 INFO MainThread:1978742 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-03 22:05:51,048 INFO MainThread:1978742 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-03 22:05:51,050 INFO MainThread:1978742 [wandb_init.py:init():847] run started, returning control to user process +2025-01-03 22:05:51,050 INFO MainThread:1978742 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/pauhv4zk/tb, True +2025-01-03 22:06:09,316 INFO MainThread:1978742 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'} +2025-01-03 22:10:26,386 WARNING MsgRouterThr:1978742 [router.py:message_loop():77] message_loop has been closed diff --git a/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/run-pauhv4zk.wandb b/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/run-pauhv4zk.wandb new file mode 100644 index 0000000000000000000000000000000000000000..c359a0f32d66bdb1a080c2f11606e1cf70b5f664 Binary files /dev/null and b/pauhv4zk/wandb/run-20250103_220550-pauhv4zk/run-pauhv4zk.wandb differ diff --git a/snapshot_6158.pt b/snapshot_6158.pt new file mode 100644 index 0000000000000000000000000000000000000000..24345b440477b8147bb7f06c04b69382758e5ee4 --- /dev/null +++ b/snapshot_6158.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb7a3b4bf2f5182483de5e13523efb4d65a9abb93eb4d40b39cdab2be56ce9c9 +size 910102870 diff --git a/sv1nnvx8/.hydra/overrides.yaml b/sv1nnvx8/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/sv1nnvx8/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/sv1nnvx8/eval_policy.log b/sv1nnvx8/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..99856a3a49c1e780798943245af7d895c39c0938 --- /dev/null +++ b/sv1nnvx8/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-04 22:35:03,328][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-04 22:35:03,331][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-04 22:35:05,321][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/tmmu86ea/episode_rosbags/episode_0_2025-01-03-20-10-01.bag b/tmmu86ea/episode_rosbags/episode_0_2025-01-03-20-10-01.bag new file mode 100644 index 0000000000000000000000000000000000000000..692c41fc7a4d99ccc129c52aadff45479dd73c1f --- /dev/null +++ b/tmmu86ea/episode_rosbags/episode_0_2025-01-03-20-10-01.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9c236f44bc909cec824ab77150821d889f1189a7414ca4225993dad0d88e5c1 +size 1411418313 diff --git a/tmmu86ea/episode_rosbags/episode_1_2025-01-03-20-11-06.bag b/tmmu86ea/episode_rosbags/episode_1_2025-01-03-20-11-06.bag new file mode 100644 index 0000000000000000000000000000000000000000..b1b9d3cec75c1decfae99a944d69ad52b100a355 --- /dev/null +++ b/tmmu86ea/episode_rosbags/episode_1_2025-01-03-20-11-06.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cff39e772f83ed0a1cae2f3f31d57c547bb5c6bdb38751ae16fa2e333a32451e +size 1413236215 diff --git a/tmmu86ea/episode_rosbags/episode_3_2025-01-03-20-12-43.bag b/tmmu86ea/episode_rosbags/episode_3_2025-01-03-20-12-43.bag new file mode 100644 index 0000000000000000000000000000000000000000..1990186d1621a2663c9cfd6ddb0abc259c229f88 --- /dev/null +++ b/tmmu86ea/episode_rosbags/episode_3_2025-01-03-20-12-43.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a15fd9b19a7b14a27f17d09601d5c491c8a6260a77cf7ffe0f3a1e6109fb34b3 +size 1411624739 diff --git a/tmmu86ea/episode_rosbags/episode_4_2025-01-03-20-13-33.bag b/tmmu86ea/episode_rosbags/episode_4_2025-01-03-20-13-33.bag new file mode 100644 index 0000000000000000000000000000000000000000..d55dcb98bc1ae2b36415e63428f4d1fc6ac30ce0 --- /dev/null +++ b/tmmu86ea/episode_rosbags/episode_4_2025-01-03-20-13-33.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:458fc56fef1cd52c72420e361601f70d8ce77596094472e77bd8666ee36f8428 +size 1413232621 diff --git a/u0i99mq0/episode_rosbags/episode_0_2025-01-04-22-40-57.bag b/u0i99mq0/episode_rosbags/episode_0_2025-01-04-22-40-57.bag new file mode 100644 index 0000000000000000000000000000000000000000..2f89ce0d46f2c0cf8f5de350094b077ee3305ff1 --- /dev/null +++ b/u0i99mq0/episode_rosbags/episode_0_2025-01-04-22-40-57.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5aa5fd6fb3f6196526f6e15d1298ff9e86999cf1778fd8a7cc5b91c37b4a3e45 +size 1408797534 diff --git a/u0i99mq0/episode_rosbags/episode_1_2025-01-04-22-41-37.bag b/u0i99mq0/episode_rosbags/episode_1_2025-01-04-22-41-37.bag new file mode 100644 index 0000000000000000000000000000000000000000..d8ef22ca5e250657367617af4f90184072e89dc6 --- /dev/null +++ b/u0i99mq0/episode_rosbags/episode_1_2025-01-04-22-41-37.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66a6b3cd639cdf059e4701d29d81b9328516465ee2f1968807bc9b3936a1c5c3 +size 1412801967 diff --git a/u0i99mq0/episode_rosbags/episode_2_2025-01-04-22-42-18.bag b/u0i99mq0/episode_rosbags/episode_2_2025-01-04-22-42-18.bag new file mode 100644 index 0000000000000000000000000000000000000000..daafb63a80406ca599c4af68a711787fad1aac64 --- /dev/null +++ b/u0i99mq0/episode_rosbags/episode_2_2025-01-04-22-42-18.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f34c7c15f144bd6db0c37165b5f9c989b76dd07d45f1188ac708912e66a1e13c +size 1409032148 diff --git a/u0i99mq0/episode_rosbags/episode_3_2025-01-04-22-49-33.bag b/u0i99mq0/episode_rosbags/episode_3_2025-01-04-22-49-33.bag new file mode 100644 index 0000000000000000000000000000000000000000..6e3f33b72e95d197a4475d6e981de9fccdb6948e --- /dev/null +++ b/u0i99mq0/episode_rosbags/episode_3_2025-01-04-22-49-33.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79bff25a917ed5617a6fd1cb7153c44f8b71f6ebfce554f4814cb0ddd12d68dd +size 1409519143 diff --git a/u0i99mq0/episode_rosbags/episode_4_2025-01-04-22-50-18.bag b/u0i99mq0/episode_rosbags/episode_4_2025-01-04-22-50-18.bag new file mode 100644 index 0000000000000000000000000000000000000000..47162607a2b161b76cf9c3f5111a4576470b2f8b --- /dev/null +++ b/u0i99mq0/episode_rosbags/episode_4_2025-01-04-22-50-18.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b124cc282adeb2fef95554d5034baaa9d57d9de28a2568597d44f53433c04eb7 +size 1408463881 diff --git a/xelb7y4j/.hydra/config.yaml b/xelb7y4j/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..30c3bfdc0a6aec7938f3f2b6ed72878e72cc1f21 --- /dev/null +++ b/xelb7y4j/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1938_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/xelb7y4j/.hydra/hydra.yaml b/xelb7y4j/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1b515753007c9b456d2bf349bc6fbd568af75caa --- /dev/null +++ b/xelb7y4j/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/xelb7y4j + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/xelb7y4j/.hydra/overrides.yaml b/xelb7y4j/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/xelb7y4j/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/xelb7y4j/eval_policy.log b/xelb7y4j/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..66c5eadd72779d8d611aede5e194405e451b335b --- /dev/null +++ b/xelb7y4j/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-03 22:05:36,907][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-03 22:05:36,910][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-03 22:05:38,030][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/ycy7lipi/.hydra/config.yaml b/ycy7lipi/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..30c3bfdc0a6aec7938f3f2b6ed72878e72cc1f21 --- /dev/null +++ b/ycy7lipi/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1938_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/ycy7lipi/.hydra/hydra.yaml b/ycy7lipi/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..66358a4a574f61f2c7ac3278717e28696bb3512b --- /dev/null +++ b/ycy7lipi/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/ycy7lipi/.hydra/overrides.yaml b/ycy7lipi/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/ycy7lipi/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/ycy7lipi/episode_rosbags/aligned_depth_to_color_K.npy b/ycy7lipi/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/ycy7lipi/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/ycy7lipi/episode_rosbags/cam_tf_world.npy b/ycy7lipi/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/ycy7lipi/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/ycy7lipi/episode_rosbags/color_K.npy b/ycy7lipi/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/ycy7lipi/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/ycy7lipi/episode_rosbags/depth_K.npy b/ycy7lipi/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/ycy7lipi/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/ycy7lipi/eval_robot.log b/ycy7lipi/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..3e68d4a56a37bf909fadb3792ad7a616441851cd --- /dev/null +++ b/ycy7lipi/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-04 21:18:58,931][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-04 21:18:58,934][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/ycy7lipi/eval_video/0_eval.mp4 b/ycy7lipi/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e7e49cfd4eecfb287fc30a856c5e094d7b5479a0 --- /dev/null +++ b/ycy7lipi/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d045a193d2f057ea903ae1b857262f56a224ec7f2b1c37cdd7f31d960664002 +size 1190484 diff --git a/ycy7lipi/eval_video/1_eval.mp4 b/ycy7lipi/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1618ca25c87b591077185671eaf72fa81dac7609 --- /dev/null +++ b/ycy7lipi/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2748fe54b33b495c534c6fda72a3d5860b022e4d001f7d76f69d9b83437382b9 +size 1196445 diff --git a/ycy7lipi/eval_video/2_eval.mp4 b/ycy7lipi/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..07cef02bc018f3f3e15517cc5374cd49ab65fa5d --- /dev/null +++ b/ycy7lipi/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd8753b9ffd389721aa4a2e54af855afb3367625e99cd27f5f88b1ef8eb8331d +size 1194301 diff --git a/ycy7lipi/eval_video/3_eval.mp4 b/ycy7lipi/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4c2a141d2aac15ec02f72a761c7e8083602720e4 --- /dev/null +++ b/ycy7lipi/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:506aff901ce34ad3ca8014f602e773afef75463b3db0e09b335ea44f384b3cfe +size 1194040 diff --git a/ycy7lipi/eval_video/4_eval.mp4 b/ycy7lipi/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9c06e4bbbd722c3941305cec5d7ec52c1cbd7534 --- /dev/null +++ b/ycy7lipi/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08c14d809d4b1c1641657c9281ba5df269caa2c3511258d59718aba1a1281ed3 +size 1210659 diff --git a/ycy7lipi/tb/events.out.tfevents.1736043545.leonmkim-ROG-Strix-G15CS-G15CS.2024840.0 b/ycy7lipi/tb/events.out.tfevents.1736043545.leonmkim-ROG-Strix-G15CS-G15CS.2024840.0 new file mode 100644 index 0000000000000000000000000000000000000000..c7cbd6fa35f17cadce59f3847f457c6263213480 --- /dev/null +++ b/ycy7lipi/tb/events.out.tfevents.1736043545.leonmkim-ROG-Strix-G15CS-G15CS.2024840.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22bea3d370311d8661fc9886181c0259da73810c01c2954ada50892c661db265 +size 6539 diff --git a/ycy7lipi/wandb/debug-internal.log b/ycy7lipi/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..0830fca708a4508b7fa6b139e0f466b45e8a342b --- /dev/null +++ b/ycy7lipi/wandb/debug-internal.log @@ -0,0 +1,732 @@ +2025-01-04 21:19:04,607 INFO StreamThr :2025023 [internal.py:wandb_internal():85] W&B internal server running at pid: 2025023, started at: 2025-01-04 21:19:04.607560 +2025-01-04 21:19:04,608 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status +2025-01-04 21:19:04,620 INFO WriterThread:2025023 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/run-ycy7lipi.wandb +2025-01-04 21:19:04,620 DEBUG SenderThread:2025023 [sender.py:send():379] send: header +2025-01-04 21:19:04,627 DEBUG SenderThread:2025023 [sender.py:send():379] send: run +2025-01-04 21:19:04,857 INFO SenderThread:2025023 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files +2025-01-04 21:19:04,857 INFO SenderThread:2025023 [sender.py:_start_run_threads():1188] run started: ycy7lipi with start time 1736043544.60824 +2025-01-04 21:19:04,875 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: check_version +2025-01-04 21:19:04,876 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: check_version +2025-01-04 21:19:04,953 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: run_start +2025-01-04 21:19:04,958 DEBUG HandlerThread:2025023 [system_info.py:__init__():26] System info init +2025-01-04 21:19:04,958 DEBUG HandlerThread:2025023 [system_info.py:__init__():41] System info init done +2025-01-04 21:19:04,958 INFO HandlerThread:2025023 [system_monitor.py:start():194] Starting system monitor +2025-01-04 21:19:04,958 INFO SystemMonitor:2025023 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-04 21:19:04,958 INFO HandlerThread:2025023 [system_monitor.py:probe():214] Collecting system info +2025-01-04 21:19:04,959 INFO SystemMonitor:2025023 [interfaces.py:start():188] Started cpu monitoring +2025-01-04 21:19:04,959 INFO SystemMonitor:2025023 [interfaces.py:start():188] Started disk monitoring +2025-01-04 21:19:04,960 INFO SystemMonitor:2025023 [interfaces.py:start():188] Started gpu monitoring +2025-01-04 21:19:04,961 INFO SystemMonitor:2025023 [interfaces.py:start():188] Started memory monitoring +2025-01-04 21:19:04,961 INFO SystemMonitor:2025023 [interfaces.py:start():188] Started network monitoring +2025-01-04 21:19:04,972 DEBUG HandlerThread:2025023 [system_info.py:probe():152] Probing system +2025-01-04 21:19:04,973 DEBUG HandlerThread:2025023 [system_info.py:_probe_git():137] Probing git +2025-01-04 21:19:04,978 DEBUG HandlerThread:2025023 [system_info.py:_probe_git():145] Probing git done +2025-01-04 21:19:04,978 DEBUG HandlerThread:2025023 [system_info.py:probe():200] Probing system done +2025-01-04 21:19:04,978 DEBUG HandlerThread:2025023 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-05T02:19:04.972392', 'startedAt': '2025-01-05T02:19:04.594408', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '02cf07d901f3d8187a302cd51c9faa80321417ee'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3937.503875, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.959, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 663.1759071350098}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-04 21:19:04,978 INFO HandlerThread:2025023 [system_monitor.py:probe():224] Finished collecting system info +2025-01-04 21:19:04,978 INFO HandlerThread:2025023 [system_monitor.py:probe():227] Publishing system info +2025-01-04 21:19:04,978 DEBUG HandlerThread:2025023 [system_info.py:_save_code():44] Saving code +2025-01-04 21:19:04,980 DEBUG HandlerThread:2025023 [system_info.py:_save_code():67] Saving code done +2025-01-04 21:19:04,980 DEBUG HandlerThread:2025023 [system_info.py:_save_patches():84] Saving git patches +2025-01-04 21:19:05,041 DEBUG HandlerThread:2025023 [system_info.py:_save_patches():126] Saving git patches done +2025-01-04 21:19:05,042 INFO HandlerThread:2025023 [system_monitor.py:probe():229] Finished publishing system info +2025-01-04 21:19:05,045 DEBUG SenderThread:2025023 [sender.py:send():379] send: files +2025-01-04 21:19:05,045 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-04 21:19:05,045 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-04 21:19:05,045 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-04 21:19:05,282 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: python_packages +2025-01-04 21:19:05,282 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: python_packages +2025-01-04 21:19:05,283 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:05,283 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:19:05,283 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:19:05,294 INFO wandb-upload_0:2025023 [upload_job.py:push():130] Uploaded file /tmp/tmpgyo1k57gwandb/d4a4yrsz-wandb-metadata.json +2025-01-04 21:19:05,308 INFO wandb-upload_2:2025023 [upload_job.py:push():130] Uploaded file /tmp/tmpgyo1k57gwandb/y4ypmh47-diff.patch +2025-01-04 21:19:05,325 INFO HandlerThread:2025023 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/tb" + save: true +} +control { + relay_id: "e68994c24a6a4dc68409f59823d5fff9" +} +_info { + stream_id: "ycy7lipi" +} + +2025-01-04 21:19:05,325 DEBUG HandlerThread:2025023 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-04 21:19:05,366 INFO wandb-upload_1:2025023 [upload_job.py:push():130] Uploaded file /tmp/tmpgyo1k57gwandb/69k2nd7b-code/FISH/eval_robot.py +2025-01-04 21:19:05,438 DEBUG SenderThread:2025023 [sender.py:send():379] send: telemetry +2025-01-04 21:19:05,438 DEBUG SenderThread:2025023 [sender.py:send():379] send: tbrecord +2025-01-04 21:19:05,439 DEBUG SenderThread:2025023 [sender.py:send():379] send: files +2025-01-04 21:19:05,439 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file events.out.tfevents.1736043545.leonmkim-ROG-Strix-G15CS-G15CS.2024840.0 with policy live +2025-01-04 21:19:05,615 INFO wandb-upload_0:2025023 [upload_job.py:push():130] Uploaded file /tmp/tmpgyo1k57gwandb/4txyyyrd-events.out.tfevents.1736043545.leonmkim-ROG-Strix-G15CS-G15CS.2024840.0 +2025-01-04 21:19:05,859 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/requirements.txt +2025-01-04 21:19:05,859 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:19:05,859 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/diff.patch +2025-01-04 21:19:05,859 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/events.out.tfevents.1736043545.leonmkim-ROG-Strix-G15CS-G15CS.2024840.0 +2025-01-04 21:19:05,860 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/code/FISH/eval_robot.py +2025-01-04 21:19:05,860 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/wandb-metadata.json +2025-01-04 21:19:05,860 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/code +2025-01-04 21:19:05,860 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/code/FISH +2025-01-04 21:19:06,283 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:07,283 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:07,858 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:19:08,283 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:08,873 DEBUG SenderThread:2025023 [sender.py:send():379] send: config +2025-01-04 21:19:09,283 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:09,859 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:19:09,881 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:19:10,283 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:11,283 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:11,860 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:19:12,284 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:13,284 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:13,861 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:19:14,284 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:15,283 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:19:15,284 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:15,862 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:19:16,284 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:17,285 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:17,862 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:19:18,285 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:19,285 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:20,283 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:19:20,283 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:19:20,285 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:20,425 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:19:21,285 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:21,863 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:19:22,286 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:23,286 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:24,286 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:25,286 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:25,865 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:19:26,284 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:19:26,286 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:27,286 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:28,287 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:29,287 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:30,287 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:31,284 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:19:31,287 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:32,287 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:33,287 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:34,287 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:35,283 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:19:35,283 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:19:35,325 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:36,288 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:36,439 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:19:36,867 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/config.yaml +2025-01-04 21:19:37,288 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:38,288 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:39,288 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:40,288 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:41,288 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:41,869 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:19:42,288 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:42,354 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:19:43,290 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:43,869 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:19:44,290 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:45,295 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:46,295 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:47,296 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:48,284 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:19:48,296 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:49,298 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:50,298 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:50,301 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:19:50,301 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:19:51,298 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:51,529 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:19:51,530 DEBUG SenderThread:2025023 [sender.py:send():379] send: files +2025-01-04 21:19:51,531 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_3d045a193d2f057ea903.mp4 with policy now +2025-01-04 21:19:51,531 DEBUG SenderThread:2025023 [sender.py:send():379] send: history +2025-01-04 21:19:51,531 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:19:51,531 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:19:51,775 INFO wandb-upload_1:2025023 [upload_job.py:push():130] Uploaded file /tmp/tmpgyo1k57gwandb/1sphpogf-media/videos/eval/0_eval_0_3d045a193d2f057ea903.mp4 +2025-01-04 21:19:51,871 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:19:51,872 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/wandb-summary.json +2025-01-04 21:19:51,872 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval/0_eval_0_3d045a193d2f057ea903.mp4 +2025-01-04 21:19:51,872 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos +2025-01-04 21:19:51,872 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media +2025-01-04 21:19:51,872 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval +2025-01-04 21:19:52,134 INFO wandb-upload_0:2025023 [upload_job.py:push():130] Uploaded file /tmp/tmpgyo1k57gwandb/ocg9hevg-media/videos/eval/0_eval_0_3d045a193d2f057ea903.mp4 +2025-01-04 21:19:52,299 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:53,298 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:53,532 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:19:53,872 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:19:54,299 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:55,298 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:55,873 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:19:56,298 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:57,299 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:58,299 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:58,865 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:19:59,300 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:59,874 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:20:00,299 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:00,874 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/events.out.tfevents.1736043545.leonmkim-ROG-Strix-G15CS-G15CS.2024840.0 +2025-01-04 21:20:01,299 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:01,875 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:20:02,299 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:03,299 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:03,865 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:20:03,876 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:20:04,300 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:04,962 DEBUG SystemMonitor:2025023 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-04 21:20:04,962 DEBUG SenderThread:2025023 [sender.py:send():379] send: stats +2025-01-04 21:20:05,300 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:05,301 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:20:05,301 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:20:05,876 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:20:06,300 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:07,300 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:08,300 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:09,300 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:09,379 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:20:10,300 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:11,301 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:12,301 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:13,301 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:14,301 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:15,285 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:20:15,301 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:16,301 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:17,304 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:18,305 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:19,305 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:20,285 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:20:20,301 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:20:20,301 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:20:20,345 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:21,304 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:22,305 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:23,305 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:24,305 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:25,285 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:20:25,305 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:25,883 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:20:26,305 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:27,305 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:27,883 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:20:28,305 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:29,306 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:29,884 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:20:30,285 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:20:30,306 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:31,329 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:31,885 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:20:32,119 DEBUG SenderThread:2025023 [sender.py:send():379] send: files +2025-01-04 21:20:32,119 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_2748fe54b33b495c534c.mp4 with policy now +2025-01-04 21:20:32,122 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:20:32,122 DEBUG SenderThread:2025023 [sender.py:send():379] send: history +2025-01-04 21:20:32,122 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:20:32,123 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:20:32,308 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:32,361 INFO wandb-upload_1:2025023 [upload_job.py:push():130] Uploaded file /tmp/tmpgyo1k57gwandb/4t5ovhjl-media/videos/eval/1_eval_1_2748fe54b33b495c534c.mp4 +2025-01-04 21:20:32,885 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/wandb-summary.json +2025-01-04 21:20:32,885 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval/1_eval_1_2748fe54b33b495c534c.mp4 +2025-01-04 21:20:32,886 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval +2025-01-04 21:20:33,162 INFO wandb-upload_0:2025023 [upload_job.py:push():130] Uploaded file /tmp/tmpgyo1k57gwandb/buv37q6y-media/videos/eval/1_eval_1_2748fe54b33b495c534c.mp4 +2025-01-04 21:20:33,307 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:33,886 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:20:34,309 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:34,963 DEBUG SenderThread:2025023 [sender.py:send():379] send: stats +2025-01-04 21:20:35,288 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:20:35,303 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:20:35,303 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:20:35,349 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:36,314 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:37,319 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:37,887 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:20:38,312 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:39,311 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:39,888 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:20:40,311 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:40,808 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:20:40,888 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/events.out.tfevents.1736043545.leonmkim-ROG-Strix-G15CS-G15CS.2024840.0 +2025-01-04 21:20:41,126 INFO wandb-upload_1:2025023 [upload_job.py:push():130] Uploaded file /tmp/tmpgyo1k57gwandb/kkjlf36h-events.out.tfevents.1736043545.leonmkim-ROG-Strix-G15CS-G15CS.2024840.0 +2025-01-04 21:20:41,310 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:41,732 DEBUG SenderThread:2025023 [sender.py:send():379] send: history +2025-01-04 21:20:41,733 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:20:41,733 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:20:41,888 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:20:41,889 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/wandb-summary.json +2025-01-04 21:20:42,310 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:43,311 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:43,889 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:20:44,313 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:45,311 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:45,890 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:20:46,286 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:20:46,311 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:47,311 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:48,311 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:49,311 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:50,301 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:20:50,302 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:20:50,349 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:51,311 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:51,404 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:20:52,311 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:53,312 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:54,312 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:55,312 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:55,893 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:20:56,312 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:57,286 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:20:57,313 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:58,312 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:59,313 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:00,313 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:01,313 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:02,313 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:02,793 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:21:03,313 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:03,896 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:21:04,313 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:04,963 DEBUG SenderThread:2025023 [sender.py:send():379] send: stats +2025-01-04 21:21:05,302 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:21:05,302 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:21:05,345 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:05,897 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:21:06,314 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:07,314 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:08,314 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:08,421 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:21:09,314 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:10,314 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:11,315 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:11,899 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:21:12,317 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:12,858 DEBUG SenderThread:2025023 [sender.py:send():379] send: files +2025-01-04 21:21:12,858 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_dd8753b9ffd389721aa4.mp4 with policy now +2025-01-04 21:21:12,858 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:21:12,859 DEBUG SenderThread:2025023 [sender.py:send():379] send: history +2025-01-04 21:21:12,859 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:21:12,859 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:21:12,899 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/wandb-summary.json +2025-01-04 21:21:12,899 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval/2_eval_2_dd8753b9ffd389721aa4.mp4 +2025-01-04 21:21:12,900 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval +2025-01-04 21:21:13,086 INFO wandb-upload_0:2025023 [upload_job.py:push():130] Uploaded file /tmp/tmpgyo1k57gwandb/dwlh7bie-media/videos/eval/2_eval_2_dd8753b9ffd389721aa4.mp4 +2025-01-04 21:21:13,316 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:13,337 INFO wandb-upload_1:2025023 [upload_job.py:push():130] Uploaded file /tmp/tmpgyo1k57gwandb/1s5hjoh9-media/videos/eval/2_eval_2_dd8753b9ffd389721aa4.mp4 +2025-01-04 21:21:13,873 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:21:13,900 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:21:14,316 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:14,900 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:21:15,322 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:16,327 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:17,326 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:17,901 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:21:18,326 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:19,272 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:21:19,321 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:20,306 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:21:20,306 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:21:20,349 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:21,324 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:21,562 DEBUG SenderThread:2025023 [sender.py:send():379] send: history +2025-01-04 21:21:21,562 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:21:21,562 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:21:21,902 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/events.out.tfevents.1736043545.leonmkim-ROG-Strix-G15CS-G15CS.2024840.0 +2025-01-04 21:21:21,903 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:21:21,903 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/wandb-summary.json +2025-01-04 21:21:22,326 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:23,323 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:23,903 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:21:24,323 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:24,514 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:21:25,385 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:26,385 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:27,385 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:27,905 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:21:28,385 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:29,385 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:29,670 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:21:30,385 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:31,385 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:32,385 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:33,386 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:34,386 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:34,964 DEBUG SenderThread:2025023 [sender.py:send():379] send: stats +2025-01-04 21:21:34,964 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:21:35,304 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:21:35,304 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:21:35,386 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:36,386 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:37,386 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:38,386 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:39,386 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:40,384 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:21:40,386 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:41,386 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:42,386 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:43,387 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:44,387 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:45,384 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:21:45,387 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:46,387 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:47,387 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:47,912 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:21:48,387 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:49,387 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:49,912 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:21:50,304 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:21:50,304 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:21:50,387 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:50,443 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:21:51,388 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:52,413 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:53,392 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:53,666 DEBUG SenderThread:2025023 [sender.py:send():379] send: files +2025-01-04 21:21:53,666 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_506aff901ce34ad3ca80.mp4 with policy now +2025-01-04 21:21:53,667 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:21:53,667 DEBUG SenderThread:2025023 [sender.py:send():379] send: history +2025-01-04 21:21:53,667 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:21:53,668 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:21:53,906 INFO wandb-upload_0:2025023 [upload_job.py:push():130] Uploaded file /tmp/tmpgyo1k57gwandb/cgshmvee-media/videos/eval/3_eval_3_506aff901ce34ad3ca80.mp4 +2025-01-04 21:21:53,914 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:21:53,914 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/wandb-summary.json +2025-01-04 21:21:53,914 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval/3_eval_3_506aff901ce34ad3ca80.mp4 +2025-01-04 21:21:53,914 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval +2025-01-04 21:21:54,163 INFO wandb-upload_1:2025023 [upload_job.py:push():130] Uploaded file /tmp/tmpgyo1k57gwandb/igfh8wt9-media/videos/eval/3_eval_3_506aff901ce34ad3ca80.mp4 +2025-01-04 21:21:54,391 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:54,914 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:21:55,393 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:55,914 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:21:56,391 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:21:56,399 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:57,398 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:58,404 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:59,409 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:59,916 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:22:00,397 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:01,401 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:01,916 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:22:02,385 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:22:02,399 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:03,397 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:03,614 DEBUG SenderThread:2025023 [sender.py:send():379] send: history +2025-01-04 21:22:03,614 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:22:03,614 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:22:03,917 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/events.out.tfevents.1736043545.leonmkim-ROG-Strix-G15CS-G15CS.2024840.0 +2025-01-04 21:22:03,917 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/wandb-summary.json +2025-01-04 21:22:04,160 INFO wandb-upload_0:2025023 [upload_job.py:push():130] Uploaded file /tmp/tmpgyo1k57gwandb/g2v5sybh-events.out.tfevents.1736043545.leonmkim-ROG-Strix-G15CS-G15CS.2024840.0 +2025-01-04 21:22:04,404 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:04,964 DEBUG SenderThread:2025023 [sender.py:send():379] send: stats +2025-01-04 21:22:05,306 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:22:05,307 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:22:05,403 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:05,918 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:22:06,400 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:07,393 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:22:07,408 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:07,919 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:22:08,405 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:09,405 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:09,919 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:22:10,406 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:11,406 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:12,406 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:13,387 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:22:13,406 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:14,406 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:15,407 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:16,408 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:17,408 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:18,387 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:22:18,408 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:19,408 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:20,305 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:22:20,305 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:22:20,409 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:21,409 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:22,409 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:23,409 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:23,442 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:22:24,409 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:25,409 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:26,409 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:27,410 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:27,925 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:22:28,410 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:29,265 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:22:29,410 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:30,410 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:31,410 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:32,410 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:33,410 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:33,927 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:22:34,412 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:34,941 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:22:34,965 DEBUG SenderThread:2025023 [sender.py:send():379] send: stats +2025-01-04 21:22:35,320 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:22:35,320 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:22:35,414 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:35,470 DEBUG SenderThread:2025023 [sender.py:send():379] send: files +2025-01-04 21:22:35,470 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_08c14d809d4b1c164165.mp4 with policy now +2025-01-04 21:22:35,471 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:22:35,471 DEBUG SenderThread:2025023 [sender.py:send():379] send: history +2025-01-04 21:22:35,472 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:22:35,472 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:22:35,697 INFO wandb-upload_1:2025023 [upload_job.py:push():130] Uploaded file /tmp/tmpgyo1k57gwandb/dwadwg54-media/videos/eval/4_eval_4_08c14d809d4b1c164165.mp4 +2025-01-04 21:22:35,929 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:22:35,929 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/wandb-summary.json +2025-01-04 21:22:35,929 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval/4_eval_4_08c14d809d4b1c164165.mp4 +2025-01-04 21:22:35,929 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval +2025-01-04 21:22:36,199 INFO wandb-upload_0:2025023 [upload_job.py:push():130] Uploaded file /tmp/tmpgyo1k57gwandb/cfmltl5f-media/videos/eval/4_eval_4_08c14d809d4b1c164165.mp4 +2025-01-04 21:22:36,422 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:37,421 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:37,929 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:22:38,427 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:39,427 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:39,930 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:22:40,153 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:22:40,426 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:41,435 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:42,435 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:43,429 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:43,931 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:22:44,424 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:44,664 DEBUG SenderThread:2025023 [sender.py:send():379] send: history +2025-01-04 21:22:44,664 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:22:44,665 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:22:44,932 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/events.out.tfevents.1736043545.leonmkim-ROG-Strix-G15CS-G15CS.2024840.0 +2025-01-04 21:22:44,932 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/wandb-summary.json +2025-01-04 21:22:45,128 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 21:22:45,129 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 21:22:45,270 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:22:45,426 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:45,626 INFO wandb-upload_1:2025023 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmphu7d25_1 +2025-01-04 21:22:45,672 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 21:22:45,682 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:22:45,932 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/events.out.tfevents.1736043545.leonmkim-ROG-Strix-G15CS-G15CS.2024840.0 +2025-01-04 21:22:45,932 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:22:45,932 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/table/eval/total_success_rate_ci_6_05144a46373be5818354.table.json +2025-01-04 21:22:45,932 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-04 21:22:45,933 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media +2025-01-04 21:22:45,933 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/table +2025-01-04 21:22:45,933 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/table/eval +2025-01-04 21:22:46,004 INFO SenderThread:2025023 [sender.py:send_request_log_artifact():1525] logged artifact run-ycy7lipi-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE0Nzk1MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1NTIxNw==', 'latestArtifact': None}} +2025-01-04 21:22:46,004 DEBUG SenderThread:2025023 [sender.py:send():379] send: files +2025-01-04 21:22:46,004 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-04 21:22:46,004 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:22:46,005 DEBUG SenderThread:2025023 [sender.py:send():379] send: history +2025-01-04 21:22:46,005 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:22:46,005 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:22:46,006 DEBUG SenderThread:2025023 [sender.py:send():379] send: telemetry +2025-01-04 21:22:46,006 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:22:46,010 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:22:46,010 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:22:46,010 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:22:46,010 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:22:46,011 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:22:46,011 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:22:46,011 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:22:46,011 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:22:46,011 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:22:46,012 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:22:46,012 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:22:46,012 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:22:46,012 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:22:46,013 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:22:46,013 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:22:46,013 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:22:46,013 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:22:46,014 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:22:46,014 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:22:46,014 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:22:46,014 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:22:46,014 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 21:22:46,188 INFO wandb-upload_2:2025023 [upload_job.py:push():130] Uploaded file /tmp/tmpgyo1k57gwandb/fs8zoofs-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-04 21:22:46,200 INFO wandb-upload_0:2025023 [upload_job.py:push():130] Uploaded file /tmp/tmpgyo1k57gwandb/au66uxn3-media/table/eval/total_success_rate_ci_6_05144a46373be5818354.table.json +2025-01-04 21:22:46,403 INFO wandb-upload_1:2025023 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp45im8dm7 +2025-01-04 21:22:46,425 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:46,454 INFO wandb-upload_2:2025023 [upload_job.py:push():130] Uploaded file /tmp/tmpgyo1k57gwandb/l3kbj1u2-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-04 21:22:46,805 INFO SenderThread:2025023 [sender.py:send_request_log_artifact():1525] logged artifact run-ycy7lipi-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE0Nzk2Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1NTIyMg==', 'latestArtifact': None}} +2025-01-04 21:22:46,805 DEBUG SenderThread:2025023 [sender.py:send():379] send: files +2025-01-04 21:22:46,805 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_05144a46373be5818354.table.json with policy now +2025-01-04 21:22:46,805 DEBUG SenderThread:2025023 [sender.py:send():379] send: history +2025-01-04 21:22:46,805 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:22:46,805 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:22:46,932 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/wandb-summary.json +2025-01-04 21:22:47,055 INFO wandb-upload_0:2025023 [upload_job.py:push():130] Uploaded file /tmp/tmpgyo1k57gwandb/yrvsuceh-media/table/eval/total_success_rate_ci_6_05144a46373be5818354.table.json +2025-01-04 21:22:47,425 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:47,933 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:22:48,425 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:49,425 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:50,307 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:22:50,307 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:22:50,425 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:51,401 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:22:51,425 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:51,935 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/config.yaml +2025-01-04 21:22:52,425 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:53,426 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:54,426 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:55,426 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:56,426 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:57,398 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:22:57,426 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:58,426 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:59,426 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:59,605 DEBUG SenderThread:2025023 [sender.py:send():379] send: exit +2025-01-04 21:22:59,606 INFO SenderThread:2025023 [sender.py:send_exit():586] handling exit code: 0 +2025-01-04 21:22:59,606 INFO SenderThread:2025023 [sender.py:send_exit():588] handling runtime: 234 +2025-01-04 21:22:59,606 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:22:59,606 INFO SenderThread:2025023 [sender.py:send_exit():594] send defer +2025-01-04 21:22:59,606 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:22:59,606 INFO HandlerThread:2025023 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-04 21:22:59,607 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: defer +2025-01-04 21:22:59,607 INFO SenderThread:2025023 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-04 21:22:59,607 INFO SenderThread:2025023 [sender.py:transition_state():614] send defer: 1 +2025-01-04 21:22:59,607 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:22:59,607 INFO HandlerThread:2025023 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-04 21:22:59,607 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: defer +2025-01-04 21:22:59,607 INFO SenderThread:2025023 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-04 21:22:59,607 INFO SenderThread:2025023 [sender.py:transition_state():614] send defer: 2 +2025-01-04 21:22:59,607 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:22:59,607 INFO HandlerThread:2025023 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-04 21:22:59,607 INFO HandlerThread:2025023 [system_monitor.py:finish():203] Stopping system monitor +2025-01-04 21:22:59,607 DEBUG SystemMonitor:2025023 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-04 21:22:59,608 DEBUG SystemMonitor:2025023 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-04 21:22:59,608 INFO HandlerThread:2025023 [interfaces.py:finish():200] Joined cpu monitor +2025-01-04 21:22:59,608 INFO HandlerThread:2025023 [interfaces.py:finish():200] Joined disk monitor +2025-01-04 21:22:59,683 INFO HandlerThread:2025023 [interfaces.py:finish():200] Joined gpu monitor +2025-01-04 21:22:59,683 INFO HandlerThread:2025023 [interfaces.py:finish():200] Joined memory monitor +2025-01-04 21:22:59,683 INFO HandlerThread:2025023 [interfaces.py:finish():200] Joined network monitor +2025-01-04 21:22:59,683 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: defer +2025-01-04 21:22:59,683 INFO SenderThread:2025023 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-04 21:22:59,683 INFO SenderThread:2025023 [sender.py:transition_state():614] send defer: 3 +2025-01-04 21:22:59,683 DEBUG SenderThread:2025023 [sender.py:send():379] send: stats +2025-01-04 21:22:59,684 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:22:59,684 INFO HandlerThread:2025023 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-04 21:22:59,684 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: defer +2025-01-04 21:22:59,684 INFO SenderThread:2025023 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-04 21:22:59,684 INFO SenderThread:2025023 [sender.py:transition_state():614] send defer: 4 +2025-01-04 21:22:59,684 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:22:59,684 INFO HandlerThread:2025023 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-04 21:22:59,938 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/wandb-summary.json +2025-01-04 21:23:01,939 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:23:06,669 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:23:06,669 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:23:06,669 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: keepalive +2025-01-04 21:23:06,671 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: defer +2025-01-04 21:23:06,671 INFO SenderThread:2025023 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-04 21:23:06,671 INFO SenderThread:2025023 [sender.py:transition_state():614] send defer: 5 +2025-01-04 21:23:06,671 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:23:06,671 DEBUG SenderThread:2025023 [sender.py:send():379] send: history +2025-01-04 21:23:06,671 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:23:06,671 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:23:06,672 INFO HandlerThread:2025023 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-04 21:23:06,672 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:23:06,673 DEBUG SenderThread:2025023 [sender.py:send():379] send: summary +2025-01-04 21:23:06,676 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:23:06,677 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: defer +2025-01-04 21:23:06,677 INFO SenderThread:2025023 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-04 21:23:06,677 INFO SenderThread:2025023 [sender.py:transition_state():614] send defer: 6 +2025-01-04 21:23:06,677 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:23:06,677 INFO HandlerThread:2025023 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-04 21:23:06,677 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: defer +2025-01-04 21:23:06,677 INFO SenderThread:2025023 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-04 21:23:06,678 INFO SenderThread:2025023 [sender.py:transition_state():614] send defer: 7 +2025-01-04 21:23:06,678 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:23:06,678 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:23:06,678 INFO HandlerThread:2025023 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-04 21:23:06,678 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: defer +2025-01-04 21:23:06,678 INFO SenderThread:2025023 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-04 21:23:06,941 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/wandb-summary.json +2025-01-04 21:23:07,607 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:23:08,973 INFO SenderThread:2025023 [sender.py:transition_state():614] send defer: 8 +2025-01-04 21:23:08,973 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:23:08,973 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:23:08,974 INFO HandlerThread:2025023 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-04 21:23:08,974 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: defer +2025-01-04 21:23:08,974 INFO SenderThread:2025023 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-04 21:23:08,974 INFO SenderThread:2025023 [job_builder.py:build():440] Attempting to build job artifact +2025-01-04 21:23:08,977 INFO SenderThread:2025023 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-04 21:23:08,979 INFO SenderThread:2025023 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-04 21:23:08,994 INFO SenderThread:2025023 [sender.py:transition_state():614] send defer: 9 +2025-01-04 21:23:08,994 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:23:08,994 INFO HandlerThread:2025023 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-04 21:23:08,994 DEBUG SenderThread:2025023 [sender.py:send():379] send: artifact +2025-01-04 21:23:09,211 INFO SenderThread:2025023 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxOTEyMTE2Ng==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTEzMDY5OQ==', 'versionIndex': 419}}} +2025-01-04 21:23:09,211 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: defer +2025-01-04 21:23:09,211 INFO SenderThread:2025023 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-04 21:23:09,212 INFO SenderThread:2025023 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-04 21:23:09,608 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:23:09,942 INFO SenderThread:2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:23:09,942 INFO SenderThread:2025023 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files +2025-01-04 21:23:09,942 INFO SenderThread:2025023 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/config.yaml config.yaml +2025-01-04 21:23:09,942 INFO SenderThread:2025023 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/wandb-summary.json wandb-summary.json +2025-01-04 21:23:09,944 INFO SenderThread:2025023 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/requirements.txt requirements.txt +2025-01-04 21:23:09,944 INFO SenderThread:2025023 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/wandb-metadata.json wandb-metadata.json +2025-01-04 21:23:09,944 INFO SenderThread:2025023 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/events.out.tfevents.1736043545.leonmkim-ROG-Strix-G15CS-G15CS.2024840.0 events.out.tfevents.1736043545.leonmkim-ROG-Strix-G15CS-G15CS.2024840.0 +2025-01-04 21:23:09,946 INFO SenderThread:2025023 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log output.log +2025-01-04 21:23:09,948 INFO SenderThread:2025023 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/diff.patch diff.patch +2025-01-04 21:23:09,948 INFO SenderThread:2025023 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-04 21:23:09,948 INFO SenderThread:2025023 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval/2_eval_2_dd8753b9ffd389721aa4.mp4 media/videos/eval/2_eval_2_dd8753b9ffd389721aa4.mp4 +2025-01-04 21:23:09,948 INFO SenderThread:2025023 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval/0_eval_0_3d045a193d2f057ea903.mp4 media/videos/eval/0_eval_0_3d045a193d2f057ea903.mp4 +2025-01-04 21:23:09,948 INFO SenderThread:2025023 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval/3_eval_3_506aff901ce34ad3ca80.mp4 media/videos/eval/3_eval_3_506aff901ce34ad3ca80.mp4 +2025-01-04 21:23:09,948 INFO SenderThread:2025023 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval/1_eval_1_2748fe54b33b495c534c.mp4 media/videos/eval/1_eval_1_2748fe54b33b495c534c.mp4 +2025-01-04 21:23:09,948 INFO SenderThread:2025023 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval/4_eval_4_08c14d809d4b1c164165.mp4 media/videos/eval/4_eval_4_08c14d809d4b1c164165.mp4 +2025-01-04 21:23:09,950 INFO SenderThread:2025023 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/table/eval/total_success_rate_ci_6_05144a46373be5818354.table.json media/table/eval/total_success_rate_ci_6_05144a46373be5818354.table.json +2025-01-04 21:23:09,950 INFO SenderThread:2025023 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-04 21:23:09,951 INFO SenderThread:2025023 [sender.py:transition_state():614] send defer: 10 +2025-01-04 21:23:09,951 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:23:09,952 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:23:09,952 INFO HandlerThread:2025023 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-04 21:23:09,952 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: defer +2025-01-04 21:23:09,952 INFO SenderThread:2025023 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-04 21:23:09,952 INFO SenderThread:2025023 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 21:23:10,126 INFO wandb-upload_1:2025023 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/requirements.txt +2025-01-04 21:23:10,138 INFO wandb-upload_2:2025023 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/config.yaml +2025-01-04 21:23:10,164 INFO wandb-upload_0:2025023 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/wandb-summary.json +2025-01-04 21:23:10,201 INFO wandb-upload_4:2025023 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:23:10,206 INFO wandb-upload_3:2025023 [upload_job.py:push():130] Uploaded file /tmp/tmpgyo1k57gwandb/83x5pxy3-events.out.tfevents.1736043545.leonmkim-ROG-Strix-G15CS-G15CS.2024840.0 +2025-01-04 21:23:10,407 INFO Thread-11 (_thread_body):2025023 [sender.py:transition_state():614] send defer: 11 +2025-01-04 21:23:10,407 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:23:10,407 INFO HandlerThread:2025023 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-04 21:23:10,407 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: defer +2025-01-04 21:23:10,408 INFO SenderThread:2025023 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-04 21:23:10,408 INFO SenderThread:2025023 [file_pusher.py:join():175] waiting for file pusher +2025-01-04 21:23:10,409 INFO SenderThread:2025023 [sender.py:transition_state():614] send defer: 12 +2025-01-04 21:23:10,409 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:23:10,409 INFO HandlerThread:2025023 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-04 21:23:10,409 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: defer +2025-01-04 21:23:10,409 INFO SenderThread:2025023 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-04 21:23:10,409 INFO SenderThread:2025023 [file_stream.py:finish():601] file stream finish called +2025-01-04 21:23:10,465 INFO SenderThread:2025023 [file_stream.py:finish():605] file stream finish is done +2025-01-04 21:23:10,465 INFO SenderThread:2025023 [sender.py:transition_state():614] send defer: 13 +2025-01-04 21:23:10,466 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:23:10,466 INFO HandlerThread:2025023 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-04 21:23:10,466 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: defer +2025-01-04 21:23:10,466 INFO SenderThread:2025023 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-04 21:23:10,466 INFO SenderThread:2025023 [sender.py:transition_state():614] send defer: 14 +2025-01-04 21:23:10,466 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:23:10,466 INFO HandlerThread:2025023 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-04 21:23:10,467 DEBUG SenderThread:2025023 [sender.py:send():379] send: final +2025-01-04 21:23:10,467 DEBUG SenderThread:2025023 [sender.py:send():379] send: footer +2025-01-04 21:23:10,467 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: defer +2025-01-04 21:23:10,467 INFO SenderThread:2025023 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-04 21:23:10,467 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:23:10,468 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: server_info +2025-01-04 21:23:10,468 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:23:10,468 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: server_info +2025-01-04 21:23:10,471 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:23:10,472 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: get_summary +2025-01-04 21:23:10,472 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-04 21:23:10,473 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:10,521 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:23:10,522 INFO MainThread:2025023 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-04 21:23:10,522 INFO MainThread:2025023 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-04 21:23:10,522 INFO MainThread:2025023 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-04 21:23:10,522 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: shutdown +2025-01-04 21:23:10,522 INFO HandlerThread:2025023 [handler.py:finish():882] shutting down handler +2025-01-04 21:23:11,472 INFO WriterThread:2025023 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/run-ycy7lipi.wandb +2025-01-04 21:23:11,522 INFO SenderThread:2025023 [sender.py:finish():1615] shutting down sender +2025-01-04 21:23:11,522 INFO SenderThread:2025023 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 21:23:11,522 INFO SenderThread:2025023 [file_pusher.py:join():175] waiting for file pusher diff --git a/ycy7lipi/wandb/debug.log b/ycy7lipi/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..c46a48b4112c3ea8526322d29e04db170359eaa2 --- /dev/null +++ b/ycy7lipi/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-04 21:19:04,600 INFO MainThread:2024840 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-04 21:19:04,601 INFO MainThread:2024840 [wandb_setup.py:_flush():76] Configure stats pid to 2024840 +2025-01-04 21:19:04,601 INFO MainThread:2024840 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-04 21:19:04,601 INFO MainThread:2024840 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/settings +2025-01-04 21:19:04,601 INFO MainThread:2024840 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-04 21:19:04,601 INFO MainThread:2024840 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-04 21:19:04,601 INFO MainThread:2024840 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-04 21:19:04,601 INFO MainThread:2024840 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-04 21:19:04,601 INFO MainThread:2024840 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/logs/debug.log +2025-01-04 21:19:04,601 INFO MainThread:2024840 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/logs/debug-internal.log +2025-01-04 21:19:04,601 INFO MainThread:2024840 [wandb_init.py:init():569] calling init triggers +2025-01-04 21:19:04,601 INFO MainThread:2024840 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1938_0', 'wandb_notes': '1938_0_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi'} +2025-01-04 21:19:04,602 INFO MainThread:2024840 [wandb_init.py:init():619] starting backend +2025-01-04 21:19:04,602 INFO MainThread:2024840 [wandb_init.py:init():623] setting up manager +2025-01-04 21:19:04,606 INFO MainThread:2024840 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-04 21:19:04,608 INFO MainThread:2024840 [wandb_init.py:init():631] backend started and connected +2025-01-04 21:19:04,619 INFO MainThread:2024840 [wandb_init.py:init():720] updated telemetry +2025-01-04 21:19:04,625 INFO MainThread:2024840 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-04 21:19:04,875 INFO MainThread:2024840 [wandb_run.py:_on_init():2435] communicating current version +2025-01-04 21:19:04,948 INFO MainThread:2024840 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-04 21:19:04,948 INFO MainThread:2024840 [wandb_init.py:init():804] starting run threads in backend +2025-01-04 21:19:05,283 INFO MainThread:2024840 [wandb_run.py:_console_start():2413] atexit reg +2025-01-04 21:19:05,283 INFO MainThread:2024840 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-04 21:19:05,283 INFO MainThread:2024840 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-04 21:19:05,283 INFO MainThread:2024840 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-04 21:19:05,285 INFO MainThread:2024840 [wandb_init.py:init():847] run started, returning control to user process +2025-01-04 21:19:05,285 INFO MainThread:2024840 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/tb, True +2025-01-04 21:19:08,873 INFO MainThread:2024840 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-04 21:23:11,523 WARNING MsgRouterThr:2024840 [router.py:message_loop():77] message_loop has been closed diff --git a/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/code/FISH/eval_robot.py b/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/config.yaml b/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c13db4ee0a25578a857c661c813fa376930001ab --- /dev/null +++ b/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 1 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1938_0' +wandb_notes: + desc: null + value: 1938_0_restarted_5 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736043544 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/diff.patch b/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..c70dbd91a4993732d181d425e68ce116710765ed --- /dev/null +++ b/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/diff.patch @@ -0,0 +1,56 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 57a851f..816d960 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -138,12 +138,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' ++# wandb_run_id: '1922_1' # seed 1 ++wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1006_0' ++# wandb_run_id: '1926_1' # seed 1 ++# wandb_run_id: '1930_0' # dataset shuffle seed 1 ++ ++ + + # RGBD+mask+contact(contact+DTC)+act history +-wandb_run_id: '1745_0' ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 71a5d1c..c7eadce 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -159,8 +159,10 @@ run_id_list = [ + ] + + run_id_list = [ +- # '1006_0', +- '1745_0' ++ # '1922_1', ++ # '1922_2', ++ '1926_1', ++ '1930_0', + ] + + checkpoint_epoch = 12000 +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index 6f97e26..285afc8 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,8 +330,8 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.005 +- self.move_to_rot_tolerance = 1.0 ++ self.move_to_trans_tolerance = 0.01 ++ self.move_to_rot_tolerance = 2.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) + self.move_to_target_timeout = 5.0 diff --git a/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json b/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json new file mode 100644 index 0000000000000000000000000000000000000000..abffe612675a3f5ba7b32314013cd92e261fa3b0 --- /dev/null +++ b/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.449680686007073, 1, 0.47999376058978616, 1, 0.6638181209236573, 1, 1, 1]]} \ No newline at end of file diff --git a/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/table/eval/total_success_rate_ci_6_05144a46373be5818354.table.json b/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/table/eval/total_success_rate_ci_6_05144a46373be5818354.table.json new file mode 100644 index 0000000000000000000000000000000000000000..f762859d82639dec5477727bf2e6532a4206426b --- /dev/null +++ b/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/table/eval/total_success_rate_ci_6_05144a46373be5818354.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[43, 45, 0.9555555555555556, 0.8341377376841794, 0.9955952054761087, 0.86772614710129, 0.9944009267667571, 0.8845370262092178, 0.9872875529298147, 0.8906857277416631, 0.9872872819149455]]} \ No newline at end of file diff --git a/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval/0_eval_0_3d045a193d2f057ea903.mp4 b/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval/0_eval_0_3d045a193d2f057ea903.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e7e49cfd4eecfb287fc30a856c5e094d7b5479a0 --- /dev/null +++ b/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval/0_eval_0_3d045a193d2f057ea903.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d045a193d2f057ea903ae1b857262f56a224ec7f2b1c37cdd7f31d960664002 +size 1190484 diff --git a/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval/1_eval_1_2748fe54b33b495c534c.mp4 b/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval/1_eval_1_2748fe54b33b495c534c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1618ca25c87b591077185671eaf72fa81dac7609 --- /dev/null +++ b/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval/1_eval_1_2748fe54b33b495c534c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2748fe54b33b495c534c6fda72a3d5860b022e4d001f7d76f69d9b83437382b9 +size 1196445 diff --git a/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval/2_eval_2_dd8753b9ffd389721aa4.mp4 b/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval/2_eval_2_dd8753b9ffd389721aa4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..07cef02bc018f3f3e15517cc5374cd49ab65fa5d --- /dev/null +++ b/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval/2_eval_2_dd8753b9ffd389721aa4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd8753b9ffd389721aa4a2e54af855afb3367625e99cd27f5f88b1ef8eb8331d +size 1194301 diff --git a/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval/3_eval_3_506aff901ce34ad3ca80.mp4 b/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval/3_eval_3_506aff901ce34ad3ca80.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4c2a141d2aac15ec02f72a761c7e8083602720e4 --- /dev/null +++ b/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval/3_eval_3_506aff901ce34ad3ca80.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:506aff901ce34ad3ca8014f602e773afef75463b3db0e09b335ea44f384b3cfe +size 1194040 diff --git a/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval/4_eval_4_08c14d809d4b1c164165.mp4 b/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval/4_eval_4_08c14d809d4b1c164165.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9c06e4bbbd722c3941305cec5d7ec52c1cbd7534 --- /dev/null +++ b/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval/4_eval_4_08c14d809d4b1c164165.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08c14d809d4b1c1641657c9281ba5df269caa2c3511258d59718aba1a1281ed3 +size 1210659 diff --git a/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log b/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..df1e10c00c920d63254b68b2028c723d269c614c --- /dev/null +++ b/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log @@ -0,0 +1,280 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736043548.873840]: resetting environment +[INFO] [1736043548.876931]: cleared current plan +[INFO] [1736043548.878383]: moving to home +[INFO] [1736043549.880593]: reached home +[INFO] [1736043549.882050]: reset action history +[INFO] [1736043552.591218]: environment reset +Starting episode 0 +[INFO] [1736043552.592922]: resetting environment +[INFO] [1736043552.597963]: cleared current plan +[INFO] [1736043552.599441]: moving to home +[INFO] [1736043553.601261]: reached home +[INFO] [1736043553.602619]: reset action history +[INFO] [1736043556.310130]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1736043559.753479]: resetting environment +[INFO] [1736043559.758913]: cleared current plan +[INFO] [1736043559.760304]: moving to home +[INFO] [1736043560.762997]: reached home +[INFO] [1736043560.764179]: reset action history +[INFO] [1736043563.472087]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/episode_rosbags/episode_0_2025-01-04-21-19-23.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2024840.log +started roslaunch server http://158.130.50.37:40671/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2025146] +process[print_text-2]: started with pid [2025147] +process[rosbag_record-3]: started with pid [2025148] +started bagging! +For topic gripper_width: timestamp difference is 100585920 for nearest: 1736043580351576072 - target: 1736043580250990152 at idx 228 +For topic gripper_width: timestamp difference is 102160261 for nearest: 1736043582351550863 - target: 1736043582249390602 at idx 249 +[INFO] [1736043590.515632]: Storing episode... +[rosbag_record-3] killing on exit[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736043591.265958]: Stored episode 1. +[INFO] [1736043591.266650]: Saving video... +[INFO] [1736043591.484258]: Video saved! +Starting episode 1 +[INFO] [1736043591.496896]: resetting environment +[INFO] [1736043591.506761]: cleared current plan +[INFO] [1736043591.507012]: moving to home +[INFO] [1736043595.109286]: reached home +[INFO] [1736043595.109852]: reset action history +[INFO] [1736043597.820953]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736043600.108806]: resetting environment +[INFO] [1736043600.111142]: cleared current plan +[INFO] [1736043600.111707]: moving to home +[INFO] [1736043601.113128]: reached home +[INFO] [1736043601.113527]: reset action history +[INFO] [1736043603.824709]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/episode_rosbags/episode_1_2025-01-04-21-20-03.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2024840.log +started roslaunch server http://158.130.50.37:34341/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2025268] +process[print_text-5]: started with pid [2025269] +process[rosbag_record-6]: started with pid [2025270] +started bagging! +For topic gripper_width: timestamp difference is 101684824 for nearest: 1736043623018219963 - target: 1736043622916535139 at idx 247 +For topic gripper_width: timestamp difference is 102359160 for nearest: 1736043623818217858 - target: 1736043623715858698 at idx 249 +For topic gripper_width: timestamp difference is 105050107 for nearest: 1736043627018246432 - target: 1736043626913196325 at idx 249 +[INFO] [1736043630.821377]: Storing episode...is 106850102 for nearest: 1736043629218220427 - target: 1736043629111370325 at idx 249 +[INFO] [1736043630.821377]: Storing episode...is 106850102 for nearest: 1736043629218220427 - target: 1736043629111370325 at idx 249 +[rosbag_record-6] killing on exit +[pub_text-4] killing on exit +[print_text-5] killing on exit +[INFO] [1736043631.574562]: Stored episode 2. +[INFO] [1736043631.576404]: Saving video... +[INFO] [1736043632.087420]: Video saved! +Starting episode 2 +[INFO] [1736043632.126060]: resetting environment +[INFO] [1736043632.142618]: cleared current plan +[INFO] [1736043632.144215]: moving to home +[INFO] [1736043635.750558]: reached home +[INFO] [1736043635.755473]: reset action history +[INFO] [1736043638.538100]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736043640.820304]: resetting environment +[INFO] [1736043640.832920]: cleared current plan +[INFO] [1736043640.834391]: moving to home +[INFO] [1736043641.838938]: reached home +[INFO] [1736043641.839387]: reset action history +[INFO] [1736043644.548424]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/episode_rosbags/episode_2_2025-01-04-21-20-44.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2024840.log +started roslaunch server http://158.130.50.37:35213/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2025410] +process[print_text-8]: started with pid [2025411] +process[rosbag_record-9]: started with pid [2025435] +started bagging! +For topic gripper_width: timestamp difference is -106218912 for nearest: 1736043654351553343 - target: 1736043654457772255 at idx 136 +For topic gripper_width: timestamp difference is -105924463 for nearest: 1736043654751512194 - target: 1736043654857436657 at idx 138 +For topic gripper_width: timestamp difference is -100400539 for nearest: 1736043661551550297 - target: 1736043661651950836 at idx 239 +For topic gripper_width: timestamp difference is -209501700 for nearest: 1736043664451551958 - target: 1736043664661053658 at idx 249 +For topic gripper_width: timestamp difference is 100012628 for nearest: 1736043669351550428 - target: 1736043669251537800 at idx 249 +[INFO] [1736043671.517949]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1736043672.298784]: Stored episode 3. +[INFO] [1736043672.301229]: Saving video... +[INFO] [1736043672.805221]: Video saved! +Starting episode 3 +[INFO] [1736043672.862635]: resetting environment +[INFO] [1736043672.872814]: cleared current plan +[INFO] [1736043672.873142]: moving to home +[WARN] [1736043672.864464]: Plan exhausted +[INFO] [1736043676.478309]: reached home +[INFO] [1736043676.488053]: reset action history +[INFO] [1736043679.270436]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736043681.461207]: resetting environment +[INFO] [1736043681.466755]: cleared current plan +[INFO] [1736043681.466973]: moving to home +[INFO] [1736043682.468922]: reached home +[INFO] [1736043682.473272]: reset action history +[INFO] [1736043685.407084]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/episode_rosbags/episode_3_2025-01-04-21-21-25.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2024840.log +started roslaunch server http://158.130.50.37:44783/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2025546] +process[print_text-11]: started with pid [2025547] +process[rosbag_record-12]: started with pid [2025571] +started bagging! +For topic gripper_width: timestamp difference is 135147083 for nearest: 1736043705818216312 - target: 1736043705683069229 at idx 249 +For topic gripper_width: timestamp difference is 137448405 for nearest: 1736043708618213748 - target: 1736043708480765343 at idx 249 +[INFO] [1736043712.370795]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[WARN] [1736043713.540845]: Plan exhausted +[WARN] [1736043713.578774]: Plan exhausted +[WARN] [1736043713.616236]: Plan exhausted +[WARN] [1736043713.656433]: Plan exhausted +[INFO] [1736043713.174006]: Stored episode 4. +[INFO] [1736043713.184430]: Saving video... +[INFO] [1736043713.631999]: Video saved! +Starting episode 4 +[INFO] [1736043713.669638]: resetting environment +[INFO] [1736043713.685764]: cleared current plan +[INFO] [1736043713.688906]: moving to home +[INFO] [1736043717.305350]: reached home +[INFO] [1736043717.317198]: reset action history +[INFO] [1736043720.123313]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736043723.228260]: resetting environment +[INFO] [1736043723.233546]: cleared current plan +[INFO] [1736043723.233719]: moving to home +[INFO] [1736043724.237274]: reached home +[INFO] [1736043724.244365]: reset action history +[INFO] [1736043727.029947]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/episode_rosbags/episode_4_2025-01-04-21-22-07.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2024840.log +started roslaunch server http://158.130.50.37:42787/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2025705] +process[print_text-14]: started with pid [2025706] +process[rosbag_record-15]: started with pid [2025707] +started bagging! +For topic gripper_width: timestamp difference is 101303526 for nearest: 1736043746218216606 - target: 1736043746116913080 at idx 249 +For topic gripper_width: timestamp difference is 105749171 for nearest: 1736043751818216365 - target: 1736043751712467194 at idx 249 +[INFO] [1736043754.158481]: Storing episode... +[print_text-14] killing on exit +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit +[INFO] [1736043754.940011]: Stored episode 5. +[INFO] [1736043754.952696]: Saving video... +[INFO] [1736043755.382464]: Video saved! +[INFO] [1736043755.448567]: resetting environment +[INFO] [1736043755.500266]: cleared current plan +[INFO] [1736043755.502245]: moving to home +[INFO] [1736043759.109313]: reached home +[INFO] [1736043759.120481]: reset action history +[INFO] [1736043761.999036]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/requirements.txt b/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/wandb-metadata.json b/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..885e2015b546fb0fb36284b41fc3d51c7c98da9e --- /dev/null +++ b/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-05T02:19:04.972392", + "startedAt": "2025-01-05T02:19:04.594408", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "02cf07d901f3d8187a302cd51c9faa80321417ee" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3937.503875, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.959, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 663.1759071350098 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/wandb-summary.json b/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..aebc9bda2c3275c96af5c2349e14610a8449aa38 --- /dev/null +++ b/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "3d045a193d2f057ea903ae1b857262f56a224ec7f2b1c37cdd7f31d960664002", "size": 1190484, "path": "media/videos/eval/0_eval_0_3d045a193d2f057ea903.mp4"}, "global_step": 664, "_timestamp": 1736043765.5683305, "_runtime": 220.96009063720703, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "2748fe54b33b495c534c6fda72a3d5860b022e4d001f7d76f69d9b83437382b9", "size": 1196445, "path": "media/videos/eval/1_eval_1_2748fe54b33b495c534c.mp4"}, "eval/num_success": 5.0, "episode": 45.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.4496806859970093, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.4799937605857849, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6638181209564209, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "dd8753b9ffd389721aa4a2e54af855afb3367625e99cd27f5f88b1ef8eb8331d", "size": 1194301, "path": "media/videos/eval/2_eval_2_dd8753b9ffd389721aa4.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "506aff901ce34ad3ca8014f602e773afef75463b3db0e09b335ea44f384b3cfe", "size": 1194040, "path": "media/videos/eval/3_eval_3_506aff901ce34ad3ca80.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "08c14d809d4b1c1641657c9281ba5df269caa2c3511258d59718aba1a1281ed3", "size": 1210659, "path": "media/videos/eval/4_eval_4_08c14d809d4b1c164165.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "1b420c32327aa0ba7f7504ab8676a845cd69d26134af4445f97766065563f5e1", "size": 313, "artifact_path": "wandb-client-artifact://ya4bwb82cqfaqlgkokmdd8iyxt73ajs534h5p5s2uzlhowtnr8v3lurnlmcnevpe86rtuosjcqkgrf913fgah1xeycf0o8mkg2zli8fousofwc2pmutqwwm2lgd8d3et/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://o4mt9p1xfiqeusl78xwxrkmbrjiqt3l8g24oknesax8am1a75xmchwhyei2x33awyg22hkx0khtppqxtujvp44njjx98d6em93rq37svoq18qx1a4mcpj56u8ycjbs5d:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 43, "total_num_episodes": 45, "total_success_rate": 0.9555555555555556, "total_success_rate_lb_0.01": 0.8341377376841794, "total_success_rate_ub_0.01": 0.9955952054761087, "total_success_rate_lb_0.025": 0.86772614710129, "total_success_rate_ub_0.025": 0.9944009267667571, "total_success_rate_lb_0.05": 0.8845370262092178, "total_success_rate_ub_0.05": 0.9872875529298147, "total_success_rate_lb_0.1": 0.8906857277416631, "total_success_rate_ub_0.1": 0.9872872819149455, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "05144a46373be58183546cfa7c55fac95736a76fe195d68fef0cdde89a47270b", "size": 518, "artifact_path": "wandb-client-artifact://o41s7i6qsvmturflbpvzuktktunjegxdnqkfihtcx8x95w2fhua0yee1m2w5qva4uqbymcmdvcwgtrbk6lnref8ye74vge127oe1u2d7jr2zyj8fcsfv9bf54zzqfyq5/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://jy6uze9xjvq7g96byaaq906g5xa3612eykyqtbdc28mz7rmq42ty0w9ln8jtrdlf0xfd4kwobrraw62iq8jafs0jmlisass3k8axrckfz9adeowk3flimedctb9v4fvg:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_05144a46373be5818354.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 234}, "eval/total_success_rate": 0.9555555582046509, "eval/total_success_rate_lb_0.01": 0.8341377377510071, "eval/total_success_rate_ub_0.01": 0.9955952167510986, "eval/total_success_rate_lb_0.025": 0.8677261471748352, "eval/total_success_rate_ub_0.025": 0.9944009184837341, "eval/total_success_rate_lb_0.05": 0.8845370411872864, "eval/total_success_rate_ub_0.05": 0.9872875809669495, "eval/total_success_rate_lb_0.1": 0.8906857371330261, "eval/total_success_rate_ub_0.1": 0.9872872829437256} \ No newline at end of file diff --git a/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/logs/debug-internal.log b/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..0830fca708a4508b7fa6b139e0f466b45e8a342b --- /dev/null +++ b/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/logs/debug-internal.log @@ -0,0 +1,732 @@ +2025-01-04 21:19:04,607 INFO StreamThr :2025023 [internal.py:wandb_internal():85] W&B internal server running at pid: 2025023, started at: 2025-01-04 21:19:04.607560 +2025-01-04 21:19:04,608 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status +2025-01-04 21:19:04,620 INFO WriterThread:2025023 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/run-ycy7lipi.wandb +2025-01-04 21:19:04,620 DEBUG SenderThread:2025023 [sender.py:send():379] send: header +2025-01-04 21:19:04,627 DEBUG SenderThread:2025023 [sender.py:send():379] send: run +2025-01-04 21:19:04,857 INFO SenderThread:2025023 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files +2025-01-04 21:19:04,857 INFO SenderThread:2025023 [sender.py:_start_run_threads():1188] run started: ycy7lipi with start time 1736043544.60824 +2025-01-04 21:19:04,875 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: check_version +2025-01-04 21:19:04,876 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: check_version +2025-01-04 21:19:04,953 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: run_start +2025-01-04 21:19:04,958 DEBUG HandlerThread:2025023 [system_info.py:__init__():26] System info init +2025-01-04 21:19:04,958 DEBUG HandlerThread:2025023 [system_info.py:__init__():41] System info init done +2025-01-04 21:19:04,958 INFO HandlerThread:2025023 [system_monitor.py:start():194] Starting system monitor +2025-01-04 21:19:04,958 INFO SystemMonitor:2025023 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-04 21:19:04,958 INFO HandlerThread:2025023 [system_monitor.py:probe():214] Collecting system info +2025-01-04 21:19:04,959 INFO SystemMonitor:2025023 [interfaces.py:start():188] Started cpu monitoring +2025-01-04 21:19:04,959 INFO SystemMonitor:2025023 [interfaces.py:start():188] Started disk monitoring +2025-01-04 21:19:04,960 INFO SystemMonitor:2025023 [interfaces.py:start():188] Started gpu monitoring +2025-01-04 21:19:04,961 INFO SystemMonitor:2025023 [interfaces.py:start():188] Started memory monitoring +2025-01-04 21:19:04,961 INFO SystemMonitor:2025023 [interfaces.py:start():188] Started network monitoring +2025-01-04 21:19:04,972 DEBUG HandlerThread:2025023 [system_info.py:probe():152] Probing system +2025-01-04 21:19:04,973 DEBUG HandlerThread:2025023 [system_info.py:_probe_git():137] Probing git +2025-01-04 21:19:04,978 DEBUG HandlerThread:2025023 [system_info.py:_probe_git():145] Probing git done +2025-01-04 21:19:04,978 DEBUG HandlerThread:2025023 [system_info.py:probe():200] Probing system done +2025-01-04 21:19:04,978 DEBUG HandlerThread:2025023 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-05T02:19:04.972392', 'startedAt': '2025-01-05T02:19:04.594408', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '02cf07d901f3d8187a302cd51c9faa80321417ee'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3937.503875, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.959, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 663.1759071350098}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-04 21:19:04,978 INFO HandlerThread:2025023 [system_monitor.py:probe():224] Finished collecting system info +2025-01-04 21:19:04,978 INFO HandlerThread:2025023 [system_monitor.py:probe():227] Publishing system info +2025-01-04 21:19:04,978 DEBUG HandlerThread:2025023 [system_info.py:_save_code():44] Saving code +2025-01-04 21:19:04,980 DEBUG HandlerThread:2025023 [system_info.py:_save_code():67] Saving code done +2025-01-04 21:19:04,980 DEBUG HandlerThread:2025023 [system_info.py:_save_patches():84] Saving git patches +2025-01-04 21:19:05,041 DEBUG HandlerThread:2025023 [system_info.py:_save_patches():126] Saving git patches done +2025-01-04 21:19:05,042 INFO HandlerThread:2025023 [system_monitor.py:probe():229] Finished publishing system info +2025-01-04 21:19:05,045 DEBUG SenderThread:2025023 [sender.py:send():379] send: files +2025-01-04 21:19:05,045 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-04 21:19:05,045 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-04 21:19:05,045 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-04 21:19:05,282 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: python_packages +2025-01-04 21:19:05,282 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: python_packages +2025-01-04 21:19:05,283 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:05,283 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:19:05,283 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:19:05,294 INFO wandb-upload_0:2025023 [upload_job.py:push():130] Uploaded file /tmp/tmpgyo1k57gwandb/d4a4yrsz-wandb-metadata.json +2025-01-04 21:19:05,308 INFO wandb-upload_2:2025023 [upload_job.py:push():130] Uploaded file /tmp/tmpgyo1k57gwandb/y4ypmh47-diff.patch +2025-01-04 21:19:05,325 INFO HandlerThread:2025023 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/tb" + save: true +} +control { + relay_id: "e68994c24a6a4dc68409f59823d5fff9" +} +_info { + stream_id: "ycy7lipi" +} + +2025-01-04 21:19:05,325 DEBUG HandlerThread:2025023 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-04 21:19:05,366 INFO wandb-upload_1:2025023 [upload_job.py:push():130] Uploaded file /tmp/tmpgyo1k57gwandb/69k2nd7b-code/FISH/eval_robot.py +2025-01-04 21:19:05,438 DEBUG SenderThread:2025023 [sender.py:send():379] send: telemetry +2025-01-04 21:19:05,438 DEBUG SenderThread:2025023 [sender.py:send():379] send: tbrecord +2025-01-04 21:19:05,439 DEBUG SenderThread:2025023 [sender.py:send():379] send: files +2025-01-04 21:19:05,439 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file events.out.tfevents.1736043545.leonmkim-ROG-Strix-G15CS-G15CS.2024840.0 with policy live +2025-01-04 21:19:05,615 INFO wandb-upload_0:2025023 [upload_job.py:push():130] Uploaded file /tmp/tmpgyo1k57gwandb/4txyyyrd-events.out.tfevents.1736043545.leonmkim-ROG-Strix-G15CS-G15CS.2024840.0 +2025-01-04 21:19:05,859 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/requirements.txt +2025-01-04 21:19:05,859 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:19:05,859 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/diff.patch +2025-01-04 21:19:05,859 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/events.out.tfevents.1736043545.leonmkim-ROG-Strix-G15CS-G15CS.2024840.0 +2025-01-04 21:19:05,860 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/code/FISH/eval_robot.py +2025-01-04 21:19:05,860 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/wandb-metadata.json +2025-01-04 21:19:05,860 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/code +2025-01-04 21:19:05,860 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/code/FISH +2025-01-04 21:19:06,283 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:07,283 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:07,858 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:19:08,283 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:08,873 DEBUG SenderThread:2025023 [sender.py:send():379] send: config +2025-01-04 21:19:09,283 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:09,859 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:19:09,881 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:19:10,283 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:11,283 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:11,860 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:19:12,284 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:13,284 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:13,861 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:19:14,284 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:15,283 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:19:15,284 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:15,862 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:19:16,284 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:17,285 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:17,862 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:19:18,285 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:19,285 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:20,283 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:19:20,283 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:19:20,285 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:20,425 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:19:21,285 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:21,863 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:19:22,286 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:23,286 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:24,286 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:25,286 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:25,865 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:19:26,284 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:19:26,286 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:27,286 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:28,287 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:29,287 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:30,287 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:31,284 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:19:31,287 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:32,287 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:33,287 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:34,287 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:35,283 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:19:35,283 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:19:35,325 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:36,288 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:36,439 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:19:36,867 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/config.yaml +2025-01-04 21:19:37,288 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:38,288 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:39,288 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:40,288 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:41,288 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:41,869 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:19:42,288 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:42,354 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:19:43,290 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:43,869 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:19:44,290 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:45,295 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:46,295 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:47,296 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:48,284 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:19:48,296 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:49,298 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:50,298 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:50,301 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:19:50,301 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:19:51,298 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:51,529 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:19:51,530 DEBUG SenderThread:2025023 [sender.py:send():379] send: files +2025-01-04 21:19:51,531 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_3d045a193d2f057ea903.mp4 with policy now +2025-01-04 21:19:51,531 DEBUG SenderThread:2025023 [sender.py:send():379] send: history +2025-01-04 21:19:51,531 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:19:51,531 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:19:51,775 INFO wandb-upload_1:2025023 [upload_job.py:push():130] Uploaded file /tmp/tmpgyo1k57gwandb/1sphpogf-media/videos/eval/0_eval_0_3d045a193d2f057ea903.mp4 +2025-01-04 21:19:51,871 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:19:51,872 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/wandb-summary.json +2025-01-04 21:19:51,872 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval/0_eval_0_3d045a193d2f057ea903.mp4 +2025-01-04 21:19:51,872 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos +2025-01-04 21:19:51,872 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media +2025-01-04 21:19:51,872 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval +2025-01-04 21:19:52,134 INFO wandb-upload_0:2025023 [upload_job.py:push():130] Uploaded file /tmp/tmpgyo1k57gwandb/ocg9hevg-media/videos/eval/0_eval_0_3d045a193d2f057ea903.mp4 +2025-01-04 21:19:52,299 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:53,298 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:53,532 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:19:53,872 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:19:54,299 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:55,298 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:55,873 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:19:56,298 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:57,299 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:58,299 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:58,865 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:19:59,300 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:19:59,874 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:20:00,299 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:00,874 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/events.out.tfevents.1736043545.leonmkim-ROG-Strix-G15CS-G15CS.2024840.0 +2025-01-04 21:20:01,299 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:01,875 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:20:02,299 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:03,299 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:03,865 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:20:03,876 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:20:04,300 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:04,962 DEBUG SystemMonitor:2025023 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-04 21:20:04,962 DEBUG SenderThread:2025023 [sender.py:send():379] send: stats +2025-01-04 21:20:05,300 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:05,301 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:20:05,301 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:20:05,876 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:20:06,300 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:07,300 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:08,300 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:09,300 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:09,379 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:20:10,300 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:11,301 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:12,301 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:13,301 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:14,301 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:15,285 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:20:15,301 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:16,301 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:17,304 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:18,305 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:19,305 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:20,285 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:20:20,301 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:20:20,301 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:20:20,345 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:21,304 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:22,305 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:23,305 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:24,305 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:25,285 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:20:25,305 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:25,883 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:20:26,305 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:27,305 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:27,883 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:20:28,305 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:29,306 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:29,884 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:20:30,285 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:20:30,306 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:31,329 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:31,885 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:20:32,119 DEBUG SenderThread:2025023 [sender.py:send():379] send: files +2025-01-04 21:20:32,119 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_2748fe54b33b495c534c.mp4 with policy now +2025-01-04 21:20:32,122 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:20:32,122 DEBUG SenderThread:2025023 [sender.py:send():379] send: history +2025-01-04 21:20:32,122 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:20:32,123 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:20:32,308 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:32,361 INFO wandb-upload_1:2025023 [upload_job.py:push():130] Uploaded file /tmp/tmpgyo1k57gwandb/4t5ovhjl-media/videos/eval/1_eval_1_2748fe54b33b495c534c.mp4 +2025-01-04 21:20:32,885 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/wandb-summary.json +2025-01-04 21:20:32,885 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval/1_eval_1_2748fe54b33b495c534c.mp4 +2025-01-04 21:20:32,886 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval +2025-01-04 21:20:33,162 INFO wandb-upload_0:2025023 [upload_job.py:push():130] Uploaded file /tmp/tmpgyo1k57gwandb/buv37q6y-media/videos/eval/1_eval_1_2748fe54b33b495c534c.mp4 +2025-01-04 21:20:33,307 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:33,886 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:20:34,309 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:34,963 DEBUG SenderThread:2025023 [sender.py:send():379] send: stats +2025-01-04 21:20:35,288 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:20:35,303 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:20:35,303 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:20:35,349 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:36,314 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:37,319 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:37,887 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:20:38,312 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:39,311 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:39,888 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:20:40,311 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:40,808 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:20:40,888 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/events.out.tfevents.1736043545.leonmkim-ROG-Strix-G15CS-G15CS.2024840.0 +2025-01-04 21:20:41,126 INFO wandb-upload_1:2025023 [upload_job.py:push():130] Uploaded file /tmp/tmpgyo1k57gwandb/kkjlf36h-events.out.tfevents.1736043545.leonmkim-ROG-Strix-G15CS-G15CS.2024840.0 +2025-01-04 21:20:41,310 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:41,732 DEBUG SenderThread:2025023 [sender.py:send():379] send: history +2025-01-04 21:20:41,733 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:20:41,733 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:20:41,888 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:20:41,889 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/wandb-summary.json +2025-01-04 21:20:42,310 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:43,311 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:43,889 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:20:44,313 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:45,311 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:45,890 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:20:46,286 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:20:46,311 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:47,311 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:48,311 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:49,311 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:50,301 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:20:50,302 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:20:50,349 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:51,311 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:51,404 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:20:52,311 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:53,312 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:54,312 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:55,312 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:55,893 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:20:56,312 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:57,286 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:20:57,313 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:58,312 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:20:59,313 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:00,313 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:01,313 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:02,313 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:02,793 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:21:03,313 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:03,896 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:21:04,313 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:04,963 DEBUG SenderThread:2025023 [sender.py:send():379] send: stats +2025-01-04 21:21:05,302 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:21:05,302 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:21:05,345 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:05,897 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:21:06,314 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:07,314 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:08,314 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:08,421 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:21:09,314 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:10,314 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:11,315 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:11,899 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:21:12,317 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:12,858 DEBUG SenderThread:2025023 [sender.py:send():379] send: files +2025-01-04 21:21:12,858 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_dd8753b9ffd389721aa4.mp4 with policy now +2025-01-04 21:21:12,858 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:21:12,859 DEBUG SenderThread:2025023 [sender.py:send():379] send: history +2025-01-04 21:21:12,859 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:21:12,859 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:21:12,899 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/wandb-summary.json +2025-01-04 21:21:12,899 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval/2_eval_2_dd8753b9ffd389721aa4.mp4 +2025-01-04 21:21:12,900 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval +2025-01-04 21:21:13,086 INFO wandb-upload_0:2025023 [upload_job.py:push():130] Uploaded file /tmp/tmpgyo1k57gwandb/dwlh7bie-media/videos/eval/2_eval_2_dd8753b9ffd389721aa4.mp4 +2025-01-04 21:21:13,316 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:13,337 INFO wandb-upload_1:2025023 [upload_job.py:push():130] Uploaded file /tmp/tmpgyo1k57gwandb/1s5hjoh9-media/videos/eval/2_eval_2_dd8753b9ffd389721aa4.mp4 +2025-01-04 21:21:13,873 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:21:13,900 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:21:14,316 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:14,900 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:21:15,322 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:16,327 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:17,326 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:17,901 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:21:18,326 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:19,272 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:21:19,321 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:20,306 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:21:20,306 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:21:20,349 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:21,324 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:21,562 DEBUG SenderThread:2025023 [sender.py:send():379] send: history +2025-01-04 21:21:21,562 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:21:21,562 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:21:21,902 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/events.out.tfevents.1736043545.leonmkim-ROG-Strix-G15CS-G15CS.2024840.0 +2025-01-04 21:21:21,903 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:21:21,903 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/wandb-summary.json +2025-01-04 21:21:22,326 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:23,323 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:23,903 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:21:24,323 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:24,514 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:21:25,385 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:26,385 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:27,385 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:27,905 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:21:28,385 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:29,385 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:29,670 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:21:30,385 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:31,385 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:32,385 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:33,386 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:34,386 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:34,964 DEBUG SenderThread:2025023 [sender.py:send():379] send: stats +2025-01-04 21:21:34,964 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:21:35,304 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:21:35,304 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:21:35,386 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:36,386 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:37,386 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:38,386 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:39,386 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:40,384 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:21:40,386 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:41,386 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:42,386 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:43,387 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:44,387 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:45,384 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:21:45,387 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:46,387 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:47,387 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:47,912 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:21:48,387 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:49,387 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:49,912 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:21:50,304 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:21:50,304 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:21:50,387 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:50,443 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:21:51,388 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:52,413 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:53,392 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:53,666 DEBUG SenderThread:2025023 [sender.py:send():379] send: files +2025-01-04 21:21:53,666 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_506aff901ce34ad3ca80.mp4 with policy now +2025-01-04 21:21:53,667 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:21:53,667 DEBUG SenderThread:2025023 [sender.py:send():379] send: history +2025-01-04 21:21:53,667 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:21:53,668 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:21:53,906 INFO wandb-upload_0:2025023 [upload_job.py:push():130] Uploaded file /tmp/tmpgyo1k57gwandb/cgshmvee-media/videos/eval/3_eval_3_506aff901ce34ad3ca80.mp4 +2025-01-04 21:21:53,914 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:21:53,914 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/wandb-summary.json +2025-01-04 21:21:53,914 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval/3_eval_3_506aff901ce34ad3ca80.mp4 +2025-01-04 21:21:53,914 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval +2025-01-04 21:21:54,163 INFO wandb-upload_1:2025023 [upload_job.py:push():130] Uploaded file /tmp/tmpgyo1k57gwandb/igfh8wt9-media/videos/eval/3_eval_3_506aff901ce34ad3ca80.mp4 +2025-01-04 21:21:54,391 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:54,914 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:21:55,393 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:55,914 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:21:56,391 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:21:56,399 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:57,398 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:58,404 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:59,409 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:21:59,916 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:22:00,397 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:01,401 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:01,916 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:22:02,385 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:22:02,399 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:03,397 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:03,614 DEBUG SenderThread:2025023 [sender.py:send():379] send: history +2025-01-04 21:22:03,614 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:22:03,614 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:22:03,917 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/events.out.tfevents.1736043545.leonmkim-ROG-Strix-G15CS-G15CS.2024840.0 +2025-01-04 21:22:03,917 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/wandb-summary.json +2025-01-04 21:22:04,160 INFO wandb-upload_0:2025023 [upload_job.py:push():130] Uploaded file /tmp/tmpgyo1k57gwandb/g2v5sybh-events.out.tfevents.1736043545.leonmkim-ROG-Strix-G15CS-G15CS.2024840.0 +2025-01-04 21:22:04,404 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:04,964 DEBUG SenderThread:2025023 [sender.py:send():379] send: stats +2025-01-04 21:22:05,306 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:22:05,307 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:22:05,403 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:05,918 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:22:06,400 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:07,393 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:22:07,408 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:07,919 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:22:08,405 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:09,405 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:09,919 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:22:10,406 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:11,406 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:12,406 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:13,387 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:22:13,406 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:14,406 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:15,407 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:16,408 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:17,408 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:18,387 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:22:18,408 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:19,408 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:20,305 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:22:20,305 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:22:20,409 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:21,409 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:22,409 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:23,409 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:23,442 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:22:24,409 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:25,409 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:26,409 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:27,410 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:27,925 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:22:28,410 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:29,265 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:22:29,410 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:30,410 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:31,410 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:32,410 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:33,410 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:33,927 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:22:34,412 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:34,941 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:22:34,965 DEBUG SenderThread:2025023 [sender.py:send():379] send: stats +2025-01-04 21:22:35,320 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:22:35,320 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:22:35,414 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:35,470 DEBUG SenderThread:2025023 [sender.py:send():379] send: files +2025-01-04 21:22:35,470 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_08c14d809d4b1c164165.mp4 with policy now +2025-01-04 21:22:35,471 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:22:35,471 DEBUG SenderThread:2025023 [sender.py:send():379] send: history +2025-01-04 21:22:35,472 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:22:35,472 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:22:35,697 INFO wandb-upload_1:2025023 [upload_job.py:push():130] Uploaded file /tmp/tmpgyo1k57gwandb/dwadwg54-media/videos/eval/4_eval_4_08c14d809d4b1c164165.mp4 +2025-01-04 21:22:35,929 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:22:35,929 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/wandb-summary.json +2025-01-04 21:22:35,929 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval/4_eval_4_08c14d809d4b1c164165.mp4 +2025-01-04 21:22:35,929 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval +2025-01-04 21:22:36,199 INFO wandb-upload_0:2025023 [upload_job.py:push():130] Uploaded file /tmp/tmpgyo1k57gwandb/cfmltl5f-media/videos/eval/4_eval_4_08c14d809d4b1c164165.mp4 +2025-01-04 21:22:36,422 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:37,421 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:37,929 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:22:38,427 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:39,427 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:39,930 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:22:40,153 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:22:40,426 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:41,435 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:42,435 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:43,429 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:43,931 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:22:44,424 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:44,664 DEBUG SenderThread:2025023 [sender.py:send():379] send: history +2025-01-04 21:22:44,664 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:22:44,665 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:22:44,932 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/events.out.tfevents.1736043545.leonmkim-ROG-Strix-G15CS-G15CS.2024840.0 +2025-01-04 21:22:44,932 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/wandb-summary.json +2025-01-04 21:22:45,128 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 21:22:45,129 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 21:22:45,270 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:22:45,426 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:45,626 INFO wandb-upload_1:2025023 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmphu7d25_1 +2025-01-04 21:22:45,672 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 21:22:45,682 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:22:45,932 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/events.out.tfevents.1736043545.leonmkim-ROG-Strix-G15CS-G15CS.2024840.0 +2025-01-04 21:22:45,932 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:22:45,932 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/table/eval/total_success_rate_ci_6_05144a46373be5818354.table.json +2025-01-04 21:22:45,932 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-04 21:22:45,933 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media +2025-01-04 21:22:45,933 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/table +2025-01-04 21:22:45,933 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/table/eval +2025-01-04 21:22:46,004 INFO SenderThread:2025023 [sender.py:send_request_log_artifact():1525] logged artifact run-ycy7lipi-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE0Nzk1MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1NTIxNw==', 'latestArtifact': None}} +2025-01-04 21:22:46,004 DEBUG SenderThread:2025023 [sender.py:send():379] send: files +2025-01-04 21:22:46,004 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-04 21:22:46,004 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:22:46,005 DEBUG SenderThread:2025023 [sender.py:send():379] send: history +2025-01-04 21:22:46,005 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:22:46,005 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:22:46,006 DEBUG SenderThread:2025023 [sender.py:send():379] send: telemetry +2025-01-04 21:22:46,006 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:22:46,010 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:22:46,010 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:22:46,010 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:22:46,010 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:22:46,011 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:22:46,011 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:22:46,011 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:22:46,011 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:22:46,011 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:22:46,012 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:22:46,012 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:22:46,012 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:22:46,012 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:22:46,013 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:22:46,013 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:22:46,013 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:22:46,013 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:22:46,014 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:22:46,014 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:22:46,014 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:22:46,014 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:22:46,014 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 21:22:46,188 INFO wandb-upload_2:2025023 [upload_job.py:push():130] Uploaded file /tmp/tmpgyo1k57gwandb/fs8zoofs-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-04 21:22:46,200 INFO wandb-upload_0:2025023 [upload_job.py:push():130] Uploaded file /tmp/tmpgyo1k57gwandb/au66uxn3-media/table/eval/total_success_rate_ci_6_05144a46373be5818354.table.json +2025-01-04 21:22:46,403 INFO wandb-upload_1:2025023 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp45im8dm7 +2025-01-04 21:22:46,425 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:46,454 INFO wandb-upload_2:2025023 [upload_job.py:push():130] Uploaded file /tmp/tmpgyo1k57gwandb/l3kbj1u2-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-04 21:22:46,805 INFO SenderThread:2025023 [sender.py:send_request_log_artifact():1525] logged artifact run-ycy7lipi-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE0Nzk2Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1NTIyMg==', 'latestArtifact': None}} +2025-01-04 21:22:46,805 DEBUG SenderThread:2025023 [sender.py:send():379] send: files +2025-01-04 21:22:46,805 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_05144a46373be5818354.table.json with policy now +2025-01-04 21:22:46,805 DEBUG SenderThread:2025023 [sender.py:send():379] send: history +2025-01-04 21:22:46,805 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:22:46,805 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:22:46,932 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/wandb-summary.json +2025-01-04 21:22:47,055 INFO wandb-upload_0:2025023 [upload_job.py:push():130] Uploaded file /tmp/tmpgyo1k57gwandb/yrvsuceh-media/table/eval/total_success_rate_ci_6_05144a46373be5818354.table.json +2025-01-04 21:22:47,425 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:47,933 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:22:48,425 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:49,425 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:50,307 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:22:50,307 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:22:50,425 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:51,401 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:22:51,425 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:51,935 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/config.yaml +2025-01-04 21:22:52,425 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:53,426 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:54,426 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:55,426 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:56,426 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:57,398 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:22:57,426 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:58,426 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:59,426 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:22:59,605 DEBUG SenderThread:2025023 [sender.py:send():379] send: exit +2025-01-04 21:22:59,606 INFO SenderThread:2025023 [sender.py:send_exit():586] handling exit code: 0 +2025-01-04 21:22:59,606 INFO SenderThread:2025023 [sender.py:send_exit():588] handling runtime: 234 +2025-01-04 21:22:59,606 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:22:59,606 INFO SenderThread:2025023 [sender.py:send_exit():594] send defer +2025-01-04 21:22:59,606 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:22:59,606 INFO HandlerThread:2025023 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-04 21:22:59,607 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: defer +2025-01-04 21:22:59,607 INFO SenderThread:2025023 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-04 21:22:59,607 INFO SenderThread:2025023 [sender.py:transition_state():614] send defer: 1 +2025-01-04 21:22:59,607 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:22:59,607 INFO HandlerThread:2025023 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-04 21:22:59,607 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: defer +2025-01-04 21:22:59,607 INFO SenderThread:2025023 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-04 21:22:59,607 INFO SenderThread:2025023 [sender.py:transition_state():614] send defer: 2 +2025-01-04 21:22:59,607 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:22:59,607 INFO HandlerThread:2025023 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-04 21:22:59,607 INFO HandlerThread:2025023 [system_monitor.py:finish():203] Stopping system monitor +2025-01-04 21:22:59,607 DEBUG SystemMonitor:2025023 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-04 21:22:59,608 DEBUG SystemMonitor:2025023 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-04 21:22:59,608 INFO HandlerThread:2025023 [interfaces.py:finish():200] Joined cpu monitor +2025-01-04 21:22:59,608 INFO HandlerThread:2025023 [interfaces.py:finish():200] Joined disk monitor +2025-01-04 21:22:59,683 INFO HandlerThread:2025023 [interfaces.py:finish():200] Joined gpu monitor +2025-01-04 21:22:59,683 INFO HandlerThread:2025023 [interfaces.py:finish():200] Joined memory monitor +2025-01-04 21:22:59,683 INFO HandlerThread:2025023 [interfaces.py:finish():200] Joined network monitor +2025-01-04 21:22:59,683 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: defer +2025-01-04 21:22:59,683 INFO SenderThread:2025023 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-04 21:22:59,683 INFO SenderThread:2025023 [sender.py:transition_state():614] send defer: 3 +2025-01-04 21:22:59,683 DEBUG SenderThread:2025023 [sender.py:send():379] send: stats +2025-01-04 21:22:59,684 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:22:59,684 INFO HandlerThread:2025023 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-04 21:22:59,684 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: defer +2025-01-04 21:22:59,684 INFO SenderThread:2025023 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-04 21:22:59,684 INFO SenderThread:2025023 [sender.py:transition_state():614] send defer: 4 +2025-01-04 21:22:59,684 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:22:59,684 INFO HandlerThread:2025023 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-04 21:22:59,938 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/wandb-summary.json +2025-01-04 21:23:01,939 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:23:06,669 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:23:06,669 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:23:06,669 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: keepalive +2025-01-04 21:23:06,671 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: defer +2025-01-04 21:23:06,671 INFO SenderThread:2025023 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-04 21:23:06,671 INFO SenderThread:2025023 [sender.py:transition_state():614] send defer: 5 +2025-01-04 21:23:06,671 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:23:06,671 DEBUG SenderThread:2025023 [sender.py:send():379] send: history +2025-01-04 21:23:06,671 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:23:06,671 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:23:06,672 INFO HandlerThread:2025023 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-04 21:23:06,672 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:23:06,673 DEBUG SenderThread:2025023 [sender.py:send():379] send: summary +2025-01-04 21:23:06,676 INFO SenderThread:2025023 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:23:06,677 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: defer +2025-01-04 21:23:06,677 INFO SenderThread:2025023 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-04 21:23:06,677 INFO SenderThread:2025023 [sender.py:transition_state():614] send defer: 6 +2025-01-04 21:23:06,677 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:23:06,677 INFO HandlerThread:2025023 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-04 21:23:06,677 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: defer +2025-01-04 21:23:06,677 INFO SenderThread:2025023 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-04 21:23:06,678 INFO SenderThread:2025023 [sender.py:transition_state():614] send defer: 7 +2025-01-04 21:23:06,678 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:23:06,678 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:23:06,678 INFO HandlerThread:2025023 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-04 21:23:06,678 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: defer +2025-01-04 21:23:06,678 INFO SenderThread:2025023 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-04 21:23:06,941 INFO Thread-12 :2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/wandb-summary.json +2025-01-04 21:23:07,607 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:23:08,973 INFO SenderThread:2025023 [sender.py:transition_state():614] send defer: 8 +2025-01-04 21:23:08,973 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:23:08,973 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:23:08,974 INFO HandlerThread:2025023 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-04 21:23:08,974 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: defer +2025-01-04 21:23:08,974 INFO SenderThread:2025023 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-04 21:23:08,974 INFO SenderThread:2025023 [job_builder.py:build():440] Attempting to build job artifact +2025-01-04 21:23:08,977 INFO SenderThread:2025023 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-04 21:23:08,979 INFO SenderThread:2025023 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-04 21:23:08,994 INFO SenderThread:2025023 [sender.py:transition_state():614] send defer: 9 +2025-01-04 21:23:08,994 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:23:08,994 INFO HandlerThread:2025023 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-04 21:23:08,994 DEBUG SenderThread:2025023 [sender.py:send():379] send: artifact +2025-01-04 21:23:09,211 INFO SenderThread:2025023 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxOTEyMTE2Ng==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTEzMDY5OQ==', 'versionIndex': 419}}} +2025-01-04 21:23:09,211 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: defer +2025-01-04 21:23:09,211 INFO SenderThread:2025023 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-04 21:23:09,212 INFO SenderThread:2025023 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-04 21:23:09,608 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:23:09,942 INFO SenderThread:2025023 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:23:09,942 INFO SenderThread:2025023 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files +2025-01-04 21:23:09,942 INFO SenderThread:2025023 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/config.yaml config.yaml +2025-01-04 21:23:09,942 INFO SenderThread:2025023 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/wandb-summary.json wandb-summary.json +2025-01-04 21:23:09,944 INFO SenderThread:2025023 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/requirements.txt requirements.txt +2025-01-04 21:23:09,944 INFO SenderThread:2025023 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/wandb-metadata.json wandb-metadata.json +2025-01-04 21:23:09,944 INFO SenderThread:2025023 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/events.out.tfevents.1736043545.leonmkim-ROG-Strix-G15CS-G15CS.2024840.0 events.out.tfevents.1736043545.leonmkim-ROG-Strix-G15CS-G15CS.2024840.0 +2025-01-04 21:23:09,946 INFO SenderThread:2025023 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log output.log +2025-01-04 21:23:09,948 INFO SenderThread:2025023 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/diff.patch diff.patch +2025-01-04 21:23:09,948 INFO SenderThread:2025023 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-04 21:23:09,948 INFO SenderThread:2025023 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval/2_eval_2_dd8753b9ffd389721aa4.mp4 media/videos/eval/2_eval_2_dd8753b9ffd389721aa4.mp4 +2025-01-04 21:23:09,948 INFO SenderThread:2025023 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval/0_eval_0_3d045a193d2f057ea903.mp4 media/videos/eval/0_eval_0_3d045a193d2f057ea903.mp4 +2025-01-04 21:23:09,948 INFO SenderThread:2025023 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval/3_eval_3_506aff901ce34ad3ca80.mp4 media/videos/eval/3_eval_3_506aff901ce34ad3ca80.mp4 +2025-01-04 21:23:09,948 INFO SenderThread:2025023 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval/1_eval_1_2748fe54b33b495c534c.mp4 media/videos/eval/1_eval_1_2748fe54b33b495c534c.mp4 +2025-01-04 21:23:09,948 INFO SenderThread:2025023 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/videos/eval/4_eval_4_08c14d809d4b1c164165.mp4 media/videos/eval/4_eval_4_08c14d809d4b1c164165.mp4 +2025-01-04 21:23:09,950 INFO SenderThread:2025023 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/table/eval/total_success_rate_ci_6_05144a46373be5818354.table.json media/table/eval/total_success_rate_ci_6_05144a46373be5818354.table.json +2025-01-04 21:23:09,950 INFO SenderThread:2025023 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-04 21:23:09,951 INFO SenderThread:2025023 [sender.py:transition_state():614] send defer: 10 +2025-01-04 21:23:09,951 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:23:09,952 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:23:09,952 INFO HandlerThread:2025023 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-04 21:23:09,952 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: defer +2025-01-04 21:23:09,952 INFO SenderThread:2025023 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-04 21:23:09,952 INFO SenderThread:2025023 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 21:23:10,126 INFO wandb-upload_1:2025023 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/requirements.txt +2025-01-04 21:23:10,138 INFO wandb-upload_2:2025023 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/config.yaml +2025-01-04 21:23:10,164 INFO wandb-upload_0:2025023 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/wandb-summary.json +2025-01-04 21:23:10,201 INFO wandb-upload_4:2025023 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/files/output.log +2025-01-04 21:23:10,206 INFO wandb-upload_3:2025023 [upload_job.py:push():130] Uploaded file /tmp/tmpgyo1k57gwandb/83x5pxy3-events.out.tfevents.1736043545.leonmkim-ROG-Strix-G15CS-G15CS.2024840.0 +2025-01-04 21:23:10,407 INFO Thread-11 (_thread_body):2025023 [sender.py:transition_state():614] send defer: 11 +2025-01-04 21:23:10,407 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:23:10,407 INFO HandlerThread:2025023 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-04 21:23:10,407 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: defer +2025-01-04 21:23:10,408 INFO SenderThread:2025023 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-04 21:23:10,408 INFO SenderThread:2025023 [file_pusher.py:join():175] waiting for file pusher +2025-01-04 21:23:10,409 INFO SenderThread:2025023 [sender.py:transition_state():614] send defer: 12 +2025-01-04 21:23:10,409 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:23:10,409 INFO HandlerThread:2025023 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-04 21:23:10,409 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: defer +2025-01-04 21:23:10,409 INFO SenderThread:2025023 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-04 21:23:10,409 INFO SenderThread:2025023 [file_stream.py:finish():601] file stream finish called +2025-01-04 21:23:10,465 INFO SenderThread:2025023 [file_stream.py:finish():605] file stream finish is done +2025-01-04 21:23:10,465 INFO SenderThread:2025023 [sender.py:transition_state():614] send defer: 13 +2025-01-04 21:23:10,466 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:23:10,466 INFO HandlerThread:2025023 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-04 21:23:10,466 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: defer +2025-01-04 21:23:10,466 INFO SenderThread:2025023 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-04 21:23:10,466 INFO SenderThread:2025023 [sender.py:transition_state():614] send defer: 14 +2025-01-04 21:23:10,466 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:23:10,466 INFO HandlerThread:2025023 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-04 21:23:10,467 DEBUG SenderThread:2025023 [sender.py:send():379] send: final +2025-01-04 21:23:10,467 DEBUG SenderThread:2025023 [sender.py:send():379] send: footer +2025-01-04 21:23:10,467 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: defer +2025-01-04 21:23:10,467 INFO SenderThread:2025023 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-04 21:23:10,467 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:23:10,468 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: server_info +2025-01-04 21:23:10,468 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:23:10,468 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: server_info +2025-01-04 21:23:10,471 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:23:10,472 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: get_summary +2025-01-04 21:23:10,472 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-04 21:23:10,473 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:23:10,521 DEBUG SenderThread:2025023 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:23:10,522 INFO MainThread:2025023 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-04 21:23:10,522 INFO MainThread:2025023 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-04 21:23:10,522 INFO MainThread:2025023 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-04 21:23:10,522 DEBUG HandlerThread:2025023 [handler.py:handle_request():158] handle_request: shutdown +2025-01-04 21:23:10,522 INFO HandlerThread:2025023 [handler.py:finish():882] shutting down handler +2025-01-04 21:23:11,472 INFO WriterThread:2025023 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/run-ycy7lipi.wandb +2025-01-04 21:23:11,522 INFO SenderThread:2025023 [sender.py:finish():1615] shutting down sender +2025-01-04 21:23:11,522 INFO SenderThread:2025023 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 21:23:11,522 INFO SenderThread:2025023 [file_pusher.py:join():175] waiting for file pusher diff --git a/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/logs/debug.log b/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..c46a48b4112c3ea8526322d29e04db170359eaa2 --- /dev/null +++ b/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-04 21:19:04,600 INFO MainThread:2024840 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-04 21:19:04,601 INFO MainThread:2024840 [wandb_setup.py:_flush():76] Configure stats pid to 2024840 +2025-01-04 21:19:04,601 INFO MainThread:2024840 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-04 21:19:04,601 INFO MainThread:2024840 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/settings +2025-01-04 21:19:04,601 INFO MainThread:2024840 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-04 21:19:04,601 INFO MainThread:2024840 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-04 21:19:04,601 INFO MainThread:2024840 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-04 21:19:04,601 INFO MainThread:2024840 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-04 21:19:04,601 INFO MainThread:2024840 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/logs/debug.log +2025-01-04 21:19:04,601 INFO MainThread:2024840 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/logs/debug-internal.log +2025-01-04 21:19:04,601 INFO MainThread:2024840 [wandb_init.py:init():569] calling init triggers +2025-01-04 21:19:04,601 INFO MainThread:2024840 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1938_0', 'wandb_notes': '1938_0_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi'} +2025-01-04 21:19:04,602 INFO MainThread:2024840 [wandb_init.py:init():619] starting backend +2025-01-04 21:19:04,602 INFO MainThread:2024840 [wandb_init.py:init():623] setting up manager +2025-01-04 21:19:04,606 INFO MainThread:2024840 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-04 21:19:04,608 INFO MainThread:2024840 [wandb_init.py:init():631] backend started and connected +2025-01-04 21:19:04,619 INFO MainThread:2024840 [wandb_init.py:init():720] updated telemetry +2025-01-04 21:19:04,625 INFO MainThread:2024840 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-04 21:19:04,875 INFO MainThread:2024840 [wandb_run.py:_on_init():2435] communicating current version +2025-01-04 21:19:04,948 INFO MainThread:2024840 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-04 21:19:04,948 INFO MainThread:2024840 [wandb_init.py:init():804] starting run threads in backend +2025-01-04 21:19:05,283 INFO MainThread:2024840 [wandb_run.py:_console_start():2413] atexit reg +2025-01-04 21:19:05,283 INFO MainThread:2024840 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-04 21:19:05,283 INFO MainThread:2024840 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-04 21:19:05,283 INFO MainThread:2024840 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-04 21:19:05,285 INFO MainThread:2024840 [wandb_init.py:init():847] run started, returning control to user process +2025-01-04 21:19:05,285 INFO MainThread:2024840 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1938_0/ycy7lipi/tb, True +2025-01-04 21:19:08,873 INFO MainThread:2024840 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-04 21:23:11,523 WARNING MsgRouterThr:2024840 [router.py:message_loop():77] message_loop has been closed diff --git a/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/run-ycy7lipi.wandb b/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/run-ycy7lipi.wandb new file mode 100644 index 0000000000000000000000000000000000000000..621b0326ca74ece957e7c647b2214d2f6c8b93fb Binary files /dev/null and b/ycy7lipi/wandb/run-20250104_211904-ycy7lipi/run-ycy7lipi.wandb differ diff --git a/yxg6asez/episode_rosbags/episode_0_2025-01-04-20-49-05.bag b/yxg6asez/episode_rosbags/episode_0_2025-01-04-20-49-05.bag new file mode 100644 index 0000000000000000000000000000000000000000..62cc63594e6b4ff23c282a6488e61a1cfa643147 --- /dev/null +++ b/yxg6asez/episode_rosbags/episode_0_2025-01-04-20-49-05.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ebeb749a070a526af99f078e42cbd5c9b8d7a6104ce86d4d30558827630baa7 +size 1403738572 diff --git a/yxg6asez/episode_rosbags/episode_3_2025-01-04-20-51-08.bag b/yxg6asez/episode_rosbags/episode_3_2025-01-04-20-51-08.bag new file mode 100644 index 0000000000000000000000000000000000000000..72ce37b2a7a576772ad4fea0bcc79bef5b9c895f --- /dev/null +++ b/yxg6asez/episode_rosbags/episode_3_2025-01-04-20-51-08.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4bbdc7bb595ca54e09e1974d98f98dca6704317eea111d555bd6a1ee158b2fc +size 1410164646 diff --git a/z9oszy8k/episode_rosbags/episode_4_2025-01-03-20-08-24.bag b/z9oszy8k/episode_rosbags/episode_4_2025-01-03-20-08-24.bag new file mode 100644 index 0000000000000000000000000000000000000000..34aa2f9b0c7cd0205064b6f3d00081131325506f --- /dev/null +++ b/z9oszy8k/episode_rosbags/episode_4_2025-01-03-20-08-24.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ff43655debd7e0fbd4dd705cd2bc9b5badf0674f0a4330c360ec5fb45a15f7a +size 1409162001