diff --git a/.gitattributes b/.gitattributes index fad482e08901951d197990f32862260b0e2cc5c8..4e328a41ce9b3ce49fb0d9d00e314b629f74487a 100644 --- a/.gitattributes +++ b/.gitattributes @@ -94,3 +94,49 @@ pzxh5op7/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text nl3lmlc3/episode_rosbags/episode_4_2025-01-04-21-37-56.bag filter=lfs diff=lfs merge=lfs -text nl3lmlc3/episode_rosbags/episode_1_2025-01-04-21-35-23.bag filter=lfs diff=lfs merge=lfs -text y2m69p6z/episode_rosbags/episode_3_2025-01-04-20-46-22.bag filter=lfs diff=lfs merge=lfs -text +zc0axaef/episode_rosbags/episode_1_2025-01-04-21-30-17.bag filter=lfs diff=lfs merge=lfs -text +zc0axaef/episode_rosbags/episode_2_2025-01-04-21-31-11.bag filter=lfs diff=lfs merge=lfs -text +zc0axaef/episode_rosbags/episode_0_2025-01-04-21-29-24.bag filter=lfs diff=lfs merge=lfs -text +zc0axaef/episode_rosbags/episode_3_2025-01-04-21-32-04.bag filter=lfs diff=lfs merge=lfs -text +nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval/0_eval_0_16d80c9a1dc6763cf29a.mp4 filter=lfs diff=lfs merge=lfs -text +nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval/2_eval_2_4982ced87e18fd534ea7.mp4 filter=lfs diff=lfs merge=lfs -text +nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval/4_eval_4_06b29357871c19fff932.mp4 filter=lfs diff=lfs merge=lfs -text +nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval/3_eval_3_e8c539eab0fc6faa4df8.mp4 filter=lfs diff=lfs merge=lfs -text +nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval/1_eval_1_c9d9568f97ca21d8f63a.mp4 filter=lfs diff=lfs merge=lfs -text +nl3lmlc3/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +nl3lmlc3/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +nl3lmlc3/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +nl3lmlc3/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +nl3lmlc3/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +nl3lmlc3/episode_rosbags/episode_2_2025-01-04-21-36-11.bag filter=lfs diff=lfs merge=lfs -text +nayavtp6/episode_rosbags/episode_1_2025-01-03-21-17-13.bag filter=lfs diff=lfs merge=lfs -text +nayavtp6/episode_rosbags/episode_2_2025-01-03-21-17-55.bag filter=lfs diff=lfs merge=lfs -text +zc0axaef/episode_rosbags/episode_4_2025-01-04-21-32-54.bag filter=lfs diff=lfs merge=lfs -text +nayavtp6/episode_rosbags/episode_0_2025-01-03-21-16-31.bag filter=lfs diff=lfs merge=lfs -text +nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval/1_eval_1_91c0aab1f5aa1b260cf6.mp4 filter=lfs diff=lfs merge=lfs -text +nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval/3_eval_3_a145a501509e034700f1.mp4 filter=lfs diff=lfs merge=lfs -text +nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval/4_eval_4_8ec81cfff6351c2b5db4.mp4 filter=lfs diff=lfs merge=lfs -text +nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval/2_eval_2_a653d18806cfc6db0e07.mp4 filter=lfs diff=lfs merge=lfs -text +nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval/0_eval_0_29e1a738e40422f19100.mp4 filter=lfs diff=lfs merge=lfs -text +nayavtp6/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +nayavtp6/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +nayavtp6/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +nayavtp6/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +nayavtp6/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +p96n5ha2/episode_rosbags/episode_1_2025-01-03-22-59-41.bag filter=lfs diff=lfs merge=lfs -text +nayavtp6/episode_rosbags/episode_4_2025-01-03-21-19-19.bag filter=lfs diff=lfs merge=lfs -text +iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval/4_eval_4_d691d8c6f8564711e47c.mp4 filter=lfs diff=lfs merge=lfs -text +nayavtp6/episode_rosbags/episode_3_2025-01-03-21-18-32.bag filter=lfs diff=lfs merge=lfs -text +iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval/0_eval_0_3f829840c9d5231f7c2a.mp4 filter=lfs diff=lfs merge=lfs -text +iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval/2_eval_2_fef06f9228f28f137565.mp4 filter=lfs diff=lfs merge=lfs -text +iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval/1_eval_1_fcc2c84034b2f80493e2.mp4 filter=lfs diff=lfs merge=lfs -text +iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval/3_eval_3_2ceb841f3e55434b9bc7.mp4 filter=lfs diff=lfs merge=lfs -text +iqjkgl2e/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +iqjkgl2e/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +iqjkgl2e/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +iqjkgl2e/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +iqjkgl2e/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +nl3lmlc3/episode_rosbags/episode_0_2025-01-04-21-34-30.bag filter=lfs diff=lfs merge=lfs -text +nl3lmlc3/episode_rosbags/episode_3_2025-01-04-21-37-04.bag filter=lfs diff=lfs merge=lfs -text +48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval/0_eval_0_ff4d172543858f403e01.mp4 filter=lfs diff=lfs merge=lfs -text +p96n5ha2/episode_rosbags/episode_3_2025-01-03-23-01-30.bag filter=lfs diff=lfs merge=lfs -text diff --git a/48t61yqi/.hydra/config.yaml b/48t61yqi/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5614a531a25d335cfde80ac1396030650a108114 --- /dev/null +++ b/48t61yqi/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1926_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/48t61yqi/.hydra/hydra.yaml b/48t61yqi/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..68a145abfdaac7580bb75fc5e69b60027a268a50 --- /dev/null +++ b/48t61yqi/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/48t61yqi/.hydra/overrides.yaml b/48t61yqi/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/48t61yqi/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/48t61yqi/episode_rosbags/aligned_depth_to_color_K.npy b/48t61yqi/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/48t61yqi/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/48t61yqi/episode_rosbags/cam_tf_world.npy b/48t61yqi/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/48t61yqi/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/48t61yqi/episode_rosbags/color_K.npy b/48t61yqi/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/48t61yqi/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/48t61yqi/episode_rosbags/depth_K.npy b/48t61yqi/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/48t61yqi/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/48t61yqi/episode_rosbags/episode_1_2025-01-03-21-14-59.bag b/48t61yqi/episode_rosbags/episode_1_2025-01-03-21-14-59.bag new file mode 100644 index 0000000000000000000000000000000000000000..7d16626fde9b87a2bd6a1082d3309aec549fb005 Binary files /dev/null and b/48t61yqi/episode_rosbags/episode_1_2025-01-03-21-14-59.bag differ diff --git a/48t61yqi/eval_robot.log b/48t61yqi/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..ce0e05148a0d9c750a64296117270322e53f96f2 --- /dev/null +++ b/48t61yqi/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-03 21:13:01,810][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-03 21:13:01,813][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/48t61yqi/eval_video/1_eval.mp4 b/48t61yqi/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..eba8569934431d34903d1ef46c6912e473ec97ae Binary files /dev/null and b/48t61yqi/eval_video/1_eval.mp4 differ diff --git a/48t61yqi/eval_video/2_eval.mp4 b/48t61yqi/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fa868fbb75c51fb59ae42ff30716054804074343 Binary files /dev/null and b/48t61yqi/eval_video/2_eval.mp4 differ diff --git a/48t61yqi/tb/events.out.tfevents.1735956788.leonmkim-ROG-Strix-G15CS-G15CS.1974592.0 b/48t61yqi/tb/events.out.tfevents.1735956788.leonmkim-ROG-Strix-G15CS-G15CS.1974592.0 new file mode 100644 index 0000000000000000000000000000000000000000..b0dc7109527b7da520381688278c11a835ce1de9 --- /dev/null +++ b/48t61yqi/tb/events.out.tfevents.1735956788.leonmkim-ROG-Strix-G15CS-G15CS.1974592.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:460d0b1754c3dff8072e67e31161b3fcda23ca39c7a25f6b6ffc5c5d26f1400f +size 1171 diff --git a/48t61yqi/wandb/debug-internal.log b/48t61yqi/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..71efbd4cce1a6d839a18232e1bb6e57d660e9471 --- /dev/null +++ b/48t61yqi/wandb/debug-internal.log @@ -0,0 +1,468 @@ +2025-01-03 21:13:07,574 INFO StreamThr :1974795 [internal.py:wandb_internal():85] W&B internal server running at pid: 1974795, started at: 2025-01-03 21:13:07.573731 +2025-01-03 21:13:07,576 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status +2025-01-03 21:13:07,587 INFO WriterThread:1974795 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/run-48t61yqi.wandb +2025-01-03 21:13:07,587 DEBUG SenderThread:1974795 [sender.py:send():379] send: header +2025-01-03 21:13:07,600 DEBUG SenderThread:1974795 [sender.py:send():379] send: run +2025-01-03 21:13:07,945 INFO SenderThread:1974795 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files +2025-01-03 21:13:07,945 INFO SenderThread:1974795 [sender.py:_start_run_threads():1188] run started: 48t61yqi with start time 1735956787.574555 +2025-01-03 21:13:07,961 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: check_version +2025-01-03 21:13:07,961 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: check_version +2025-01-03 21:13:08,006 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: run_start +2025-01-03 21:13:08,011 DEBUG HandlerThread:1974795 [system_info.py:__init__():26] System info init +2025-01-03 21:13:08,011 DEBUG HandlerThread:1974795 [system_info.py:__init__():41] System info init done +2025-01-03 21:13:08,011 INFO HandlerThread:1974795 [system_monitor.py:start():194] Starting system monitor +2025-01-03 21:13:08,011 INFO SystemMonitor:1974795 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-03 21:13:08,011 INFO HandlerThread:1974795 [system_monitor.py:probe():214] Collecting system info +2025-01-03 21:13:08,012 INFO SystemMonitor:1974795 [interfaces.py:start():188] Started cpu monitoring +2025-01-03 21:13:08,012 INFO SystemMonitor:1974795 [interfaces.py:start():188] Started disk monitoring +2025-01-03 21:13:08,013 INFO SystemMonitor:1974795 [interfaces.py:start():188] Started gpu monitoring +2025-01-03 21:13:08,013 INFO SystemMonitor:1974795 [interfaces.py:start():188] Started memory monitoring +2025-01-03 21:13:08,013 INFO SystemMonitor:1974795 [interfaces.py:start():188] Started network monitoring +2025-01-03 21:13:08,025 DEBUG HandlerThread:1974795 [system_info.py:probe():152] Probing system +2025-01-03 21:13:08,027 DEBUG HandlerThread:1974795 [system_info.py:_probe_git():137] Probing git +2025-01-03 21:13:08,031 DEBUG HandlerThread:1974795 [system_info.py:_probe_git():145] Probing git done +2025-01-03 21:13:08,031 DEBUG HandlerThread:1974795 [system_info.py:probe():200] Probing system done +2025-01-03 21:13:08,031 DEBUG HandlerThread:1974795 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-04T02:13:08.025575', 'startedAt': '2025-01-04T02:13:07.566812', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'c537eb6734155592809c2a7db5cca971ec186d72'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3750.002625, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.998, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.061, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 804.7593460083008}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-03 21:13:08,031 INFO HandlerThread:1974795 [system_monitor.py:probe():224] Finished collecting system info +2025-01-03 21:13:08,031 INFO HandlerThread:1974795 [system_monitor.py:probe():227] Publishing system info +2025-01-03 21:13:08,031 DEBUG HandlerThread:1974795 [system_info.py:_save_code():44] Saving code +2025-01-03 21:13:08,033 DEBUG HandlerThread:1974795 [system_info.py:_save_code():67] Saving code done +2025-01-03 21:13:08,033 DEBUG HandlerThread:1974795 [system_info.py:_save_patches():84] Saving git patches +2025-01-03 21:13:08,094 DEBUG HandlerThread:1974795 [system_info.py:_save_patches():126] Saving git patches done +2025-01-03 21:13:08,095 INFO HandlerThread:1974795 [system_monitor.py:probe():229] Finished publishing system info +2025-01-03 21:13:08,098 DEBUG SenderThread:1974795 [sender.py:send():379] send: files +2025-01-03 21:13:08,098 INFO SenderThread:1974795 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-03 21:13:08,098 INFO SenderThread:1974795 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-03 21:13:08,098 INFO SenderThread:1974795 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-03 21:13:08,340 INFO wandb-upload_2:1974795 [upload_job.py:push():130] Uploaded file /tmp/tmpm2wbqrblwandb/7zaf7m84-diff.patch +2025-01-03 21:13:08,350 INFO wandb-upload_0:1974795 [upload_job.py:push():130] Uploaded file /tmp/tmpm2wbqrblwandb/2v3neepq-wandb-metadata.json +2025-01-03 21:13:08,359 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: python_packages +2025-01-03 21:13:08,360 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: python_packages +2025-01-03 21:13:08,360 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:13:08,360 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:08,361 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:13:08,362 INFO HandlerThread:1974795 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/tb" + save: true +} +control { + relay_id: "106e59ccfedd40bd8337b716b182d46a" +} +_info { + stream_id: "48t61yqi" +} + +2025-01-03 21:13:08,362 DEBUG HandlerThread:1974795 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-03 21:13:08,373 INFO wandb-upload_1:1974795 [upload_job.py:push():130] Uploaded file /tmp/tmpm2wbqrblwandb/wfd4dtnr-code/FISH/eval_robot.py +2025-01-03 21:13:08,515 DEBUG SenderThread:1974795 [sender.py:send():379] send: telemetry +2025-01-03 21:13:08,515 DEBUG SenderThread:1974795 [sender.py:send():379] send: tbrecord +2025-01-03 21:13:08,515 DEBUG SenderThread:1974795 [sender.py:send():379] send: files +2025-01-03 21:13:08,515 INFO SenderThread:1974795 [sender.py:_save_file():1454] saving file events.out.tfevents.1735956788.leonmkim-ROG-Strix-G15CS-G15CS.1974592.0 with policy live +2025-01-03 21:13:08,698 INFO wandb-upload_2:1974795 [upload_job.py:push():130] Uploaded file /tmp/tmpm2wbqrblwandb/utluf6qk-events.out.tfevents.1735956788.leonmkim-ROG-Strix-G15CS-G15CS.1974592.0 +2025-01-03 21:13:08,946 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/requirements.txt +2025-01-03 21:13:08,946 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:13:08,947 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/code/FISH/eval_robot.py +2025-01-03 21:13:08,947 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/wandb-metadata.json +2025-01-03 21:13:08,947 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/events.out.tfevents.1735956788.leonmkim-ROG-Strix-G15CS-G15CS.1974592.0 +2025-01-03 21:13:08,947 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/diff.patch +2025-01-03 21:13:08,947 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/code/FISH +2025-01-03 21:13:08,947 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/code +2025-01-03 21:13:09,360 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:10,360 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:10,945 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:13:11,360 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:12,361 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:13,360 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:13:13,361 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:14,361 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:15,362 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:16,362 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:17,362 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:18,360 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:13:18,362 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:19,362 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:20,362 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:21,362 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:22,362 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:23,360 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:13:23,360 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:13:23,401 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:23,464 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:13:24,362 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:25,363 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:25,966 DEBUG SenderThread:1974795 [sender.py:send():379] send: config +2025-01-03 21:13:26,363 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:26,950 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:13:27,363 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:28,363 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:28,950 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:13:29,360 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:13:29,363 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:30,363 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:30,951 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:13:31,363 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:32,363 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:32,951 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:13:33,363 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:34,364 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:34,461 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:13:34,952 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:13:35,364 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:36,365 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:37,364 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:38,360 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:13:38,360 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:13:38,364 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:38,953 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:13:39,364 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:39,520 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:13:39,953 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/config.yaml +2025-01-03 21:13:40,364 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:41,365 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:42,365 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:42,954 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:13:43,365 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:44,365 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:45,361 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:13:45,365 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:46,365 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:47,365 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:48,366 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:49,365 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:50,361 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:13:50,366 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:51,366 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:52,366 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:53,360 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:13:53,360 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:13:53,401 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:54,366 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:55,366 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:55,497 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:13:56,366 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:57,366 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:58,366 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:59,367 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:00,367 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:01,362 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:14:01,367 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:02,367 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:03,367 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:04,367 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:05,367 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:06,368 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:07,313 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:14:07,368 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:08,014 DEBUG SystemMonitor:1974795 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-03 21:14:08,014 DEBUG SenderThread:1974795 [sender.py:send():379] send: stats +2025-01-03 21:14:08,361 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:14:08,361 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:14:08,409 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:08,961 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:14:09,368 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:10,368 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:11,368 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:12,368 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:12,962 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:14:13,073 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:14:13,368 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:14,372 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:15,370 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:16,371 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:16,652 DEBUG SenderThread:1974795 [sender.py:send():379] send: files +2025-01-03 21:14:16,652 INFO SenderThread:1974795 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_ff4d172543858f403e01.mp4 with policy now +2025-01-03 21:14:16,652 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:14:16,653 DEBUG SenderThread:1974795 [sender.py:send():379] send: history +2025-01-03 21:14:16,654 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:14:16,654 INFO SenderThread:1974795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:14:16,878 INFO wandb-upload_1:1974795 [upload_job.py:push():130] Uploaded file /tmp/tmpm2wbqrblwandb/kh7pplhk-media/videos/eval/0_eval_0_ff4d172543858f403e01.mp4 +2025-01-03 21:14:16,963 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:14:16,963 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval/0_eval_0_ff4d172543858f403e01.mp4 +2025-01-03 21:14:16,964 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/wandb-summary.json +2025-01-03 21:14:16,964 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos +2025-01-03 21:14:16,964 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media +2025-01-03 21:14:16,964 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval +2025-01-03 21:14:17,232 INFO wandb-upload_2:1974795 [upload_job.py:push():130] Uploaded file /tmp/tmpm2wbqrblwandb/rsn814pa-media/videos/eval/0_eval_0_ff4d172543858f403e01.mp4 +2025-01-03 21:14:17,371 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:18,362 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:14:18,372 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:18,964 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:14:19,373 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:20,409 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:21,372 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:22,374 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:22,965 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:14:23,361 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:14:23,361 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:14:23,375 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:23,479 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:14:24,374 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:24,966 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:14:25,375 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:26,374 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:27,375 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:28,374 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:29,362 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:14:29,374 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:30,375 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:31,375 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:32,375 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:33,375 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:34,375 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:35,362 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:14:35,375 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:36,376 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:37,376 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:38,015 DEBUG SenderThread:1974795 [sender.py:send():379] send: stats +2025-01-03 21:14:38,361 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:14:38,361 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:14:38,405 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:39,376 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:40,376 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:40,473 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:14:41,377 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:42,377 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:43,377 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:44,378 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:45,377 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:46,363 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:14:46,377 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:47,378 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:48,378 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:49,378 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:50,379 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:51,379 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:52,363 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:14:52,379 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:53,361 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:14:53,362 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:14:53,409 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:54,379 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:55,380 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:56,380 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:56,976 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:14:56,976 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/events.out.tfevents.1735956788.leonmkim-ROG-Strix-G15CS-G15CS.1974592.0 +2025-01-03 21:14:57,223 INFO wandb-upload_1:1974795 [upload_job.py:push():130] Uploaded file /tmp/tmpm2wbqrblwandb/2qynfcuc-events.out.tfevents.1735956788.leonmkim-ROG-Strix-G15CS-G15CS.1974592.0 +2025-01-03 21:14:57,380 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:58,181 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:14:58,380 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:58,976 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:14:59,381 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:00,380 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:00,977 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:15:01,380 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:01,533 DEBUG SenderThread:1974795 [sender.py:send():379] send: files +2025-01-03 21:15:01,533 INFO SenderThread:1974795 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_ae729244ac49b5c34423.mp4 with policy now +2025-01-03 21:15:01,533 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:15:01,534 DEBUG SenderThread:1974795 [sender.py:send():379] send: history +2025-01-03 21:15:01,534 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:15:01,535 INFO SenderThread:1974795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:15:01,978 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/wandb-summary.json +2025-01-03 21:15:01,978 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval/1_eval_1_ae729244ac49b5c34423.mp4 +2025-01-03 21:15:01,978 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval +2025-01-03 21:15:02,264 INFO wandb-upload_2:1974795 [upload_job.py:push():130] Uploaded file /tmp/tmpm2wbqrblwandb/ob86mf5s-media/videos/eval/1_eval_1_ae729244ac49b5c34423.mp4 +2025-01-03 21:15:02,389 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:02,513 INFO wandb-upload_1:1974795 [upload_job.py:push():130] Uploaded file /tmp/tmpm2wbqrblwandb/c7fg0d8r-media/videos/eval/1_eval_1_ae729244ac49b5c34423.mp4 +2025-01-03 21:15:02,978 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:15:03,365 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:15:03,389 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:04,384 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:04,979 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:15:05,383 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:06,385 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:06,979 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:15:07,384 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:08,015 DEBUG SenderThread:1974795 [sender.py:send():379] send: stats +2025-01-03 21:15:08,362 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:15:08,363 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:15:08,405 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:08,465 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:15:09,388 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:10,389 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:11,388 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:12,390 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:13,393 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:14,369 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:15:14,390 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:15,393 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:16,391 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:17,396 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:18,396 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:19,372 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:15:19,395 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:20,395 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:21,395 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:22,395 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:23,362 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:15:23,362 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:15:23,409 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:24,397 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:24,467 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:15:25,399 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:26,399 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:27,399 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:28,401 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:29,403 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:29,420 DEBUG SenderThread:1974795 [sender.py:send():379] send: files +2025-01-03 21:15:29,420 INFO SenderThread:1974795 [sender.py:_save_file():1454] saving file media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4 with policy now +2025-01-03 21:15:29,465 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:15:29,465 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: server_info +2025-01-03 21:15:29,466 DEBUG SenderThread:1974795 [sender.py:send():379] send: history +2025-01-03 21:15:29,466 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:15:29,467 INFO SenderThread:1974795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:15:29,468 DEBUG SenderThread:1974795 [sender.py:send():379] send: telemetry +2025-01-03 21:15:29,468 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: server_info +2025-01-03 21:15:29,468 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:15:29,537 DEBUG SenderThread:1974795 [sender.py:send():379] send: exit +2025-01-03 21:15:29,537 INFO SenderThread:1974795 [sender.py:send_exit():586] handling exit code: 0 +2025-01-03 21:15:29,537 INFO SenderThread:1974795 [sender.py:send_exit():588] handling runtime: 141 +2025-01-03 21:15:29,537 INFO SenderThread:1974795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:15:29,537 INFO SenderThread:1974795 [sender.py:send_exit():594] send defer +2025-01-03 21:15:29,537 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:15:29,537 INFO HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-03 21:15:29,538 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: defer +2025-01-03 21:15:29,538 INFO SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-03 21:15:29,538 INFO SenderThread:1974795 [sender.py:transition_state():614] send defer: 1 +2025-01-03 21:15:29,538 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:15:29,538 INFO HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-03 21:15:29,538 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: defer +2025-01-03 21:15:29,538 INFO SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-03 21:15:29,538 INFO SenderThread:1974795 [sender.py:transition_state():614] send defer: 2 +2025-01-03 21:15:29,538 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:15:29,538 INFO HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-03 21:15:29,538 INFO HandlerThread:1974795 [system_monitor.py:finish():203] Stopping system monitor +2025-01-03 21:15:29,538 DEBUG SystemMonitor:1974795 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-03 21:15:29,538 INFO HandlerThread:1974795 [interfaces.py:finish():200] Joined cpu monitor +2025-01-03 21:15:29,538 DEBUG SystemMonitor:1974795 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-03 21:15:29,539 INFO HandlerThread:1974795 [interfaces.py:finish():200] Joined disk monitor +2025-01-03 21:15:29,590 INFO HandlerThread:1974795 [interfaces.py:finish():200] Joined gpu monitor +2025-01-03 21:15:29,591 INFO HandlerThread:1974795 [interfaces.py:finish():200] Joined memory monitor +2025-01-03 21:15:29,591 INFO HandlerThread:1974795 [interfaces.py:finish():200] Joined network monitor +2025-01-03 21:15:29,591 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: defer +2025-01-03 21:15:29,591 INFO SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-03 21:15:29,591 INFO SenderThread:1974795 [sender.py:transition_state():614] send defer: 3 +2025-01-03 21:15:29,591 DEBUG SenderThread:1974795 [sender.py:send():379] send: stats +2025-01-03 21:15:29,591 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:15:29,591 INFO HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-03 21:15:29,592 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: defer +2025-01-03 21:15:29,592 INFO SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-03 21:15:29,592 INFO SenderThread:1974795 [sender.py:transition_state():614] send defer: 4 +2025-01-03 21:15:29,592 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:15:29,592 INFO HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-03 21:15:29,596 INFO wandb-upload_2:1974795 [upload_job.py:push():130] Uploaded file /tmp/tmpm2wbqrblwandb/44ln0e3q-media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4 +2025-01-03 21:15:29,987 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/wandb-summary.json +2025-01-03 21:15:29,987 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4 +2025-01-03 21:15:29,987 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval +2025-01-03 21:15:30,242 INFO wandb-upload_1:1974795 [upload_job.py:push():130] Uploaded file /tmp/tmpm2wbqrblwandb/vh0f04q3-media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4 +2025-01-03 21:15:30,987 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:15:36,759 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:15:36,760 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:15:36,760 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 21:15:36,760 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: defer +2025-01-03 21:15:36,761 INFO SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-03 21:15:36,761 INFO SenderThread:1974795 [sender.py:transition_state():614] send defer: 5 +2025-01-03 21:15:36,761 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:15:36,761 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:15:36,761 INFO HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-03 21:15:36,762 DEBUG SenderThread:1974795 [sender.py:send():379] send: history +2025-01-03 21:15:36,762 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:15:36,762 INFO SenderThread:1974795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:15:36,762 DEBUG SenderThread:1974795 [sender.py:send():379] send: summary +2025-01-03 21:15:36,766 INFO SenderThread:1974795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:15:36,767 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: defer +2025-01-03 21:15:36,767 INFO SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-03 21:15:36,767 INFO SenderThread:1974795 [sender.py:transition_state():614] send defer: 6 +2025-01-03 21:15:36,767 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:15:36,767 INFO HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-03 21:15:36,767 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: defer +2025-01-03 21:15:36,767 INFO SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-03 21:15:36,770 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:15:36,885 INFO SenderThread:1974795 [sender.py:transition_state():614] send defer: 7 +2025-01-03 21:15:36,885 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:15:36,885 INFO HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-03 21:15:36,885 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: defer +2025-01-03 21:15:36,885 INFO SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-03 21:15:36,989 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/config.yaml +2025-01-03 21:15:36,989 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/wandb-summary.json +2025-01-03 21:15:37,544 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:15:38,663 INFO SenderThread:1974795 [sender.py:transition_state():614] send defer: 8 +2025-01-03 21:15:38,664 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:15:38,664 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:15:38,664 INFO HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-03 21:15:38,664 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: defer +2025-01-03 21:15:38,664 INFO SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-03 21:15:38,664 INFO SenderThread:1974795 [job_builder.py:build():440] Attempting to build job artifact +2025-01-03 21:15:38,666 INFO SenderThread:1974795 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-03 21:15:38,680 INFO SenderThread:1974795 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-03 21:15:38,694 INFO SenderThread:1974795 [sender.py:transition_state():614] send defer: 9 +2025-01-03 21:15:38,694 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:15:38,694 INFO HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-03 21:15:38,694 DEBUG SenderThread:1974795 [sender.py:send():379] send: artifact +2025-01-03 21:15:38,990 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:15:39,322 INFO wandb-upload_0:1974795 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpjy2pywco +2025-01-03 21:15:39,428 INFO wandb-upload_2:1974795 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpqkmuwojg +2025-01-03 21:15:39,482 INFO wandb-upload_1:1974795 [upload_job.py:push():88] Uploaded file /tmp/tmpwr759ffl/wandb-job.json +2025-01-03 21:15:39,558 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:15:40,071 INFO SenderThread:1974795 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNzg5OTM1NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNzg3MjQ4NQ==', 'versionIndex': 408}}} +2025-01-03 21:15:40,072 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: defer +2025-01-03 21:15:40,072 INFO SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-03 21:15:40,072 INFO SenderThread:1974795 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-03 21:15:40,991 INFO SenderThread:1974795 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files +2025-01-03 21:15:40,991 INFO SenderThread:1974795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/config.yaml config.yaml +2025-01-03 21:15:40,991 INFO SenderThread:1974795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/wandb-summary.json wandb-summary.json +2025-01-03 21:15:40,991 INFO SenderThread:1974795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/requirements.txt requirements.txt +2025-01-03 21:15:40,991 INFO SenderThread:1974795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/wandb-metadata.json wandb-metadata.json +2025-01-03 21:15:40,993 INFO SenderThread:1974795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/events.out.tfevents.1735956788.leonmkim-ROG-Strix-G15CS-G15CS.1974592.0 events.out.tfevents.1735956788.leonmkim-ROG-Strix-G15CS-G15CS.1974592.0 +2025-01-03 21:15:40,996 INFO SenderThread:1974795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log output.log +2025-01-03 21:15:40,996 INFO SenderThread:1974795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/diff.patch diff.patch +2025-01-03 21:15:40,999 INFO SenderThread:1974795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-03 21:15:40,999 INFO SenderThread:1974795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4 media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4 +2025-01-03 21:15:40,999 INFO SenderThread:1974795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval/0_eval_0_ff4d172543858f403e01.mp4 media/videos/eval/0_eval_0_ff4d172543858f403e01.mp4 +2025-01-03 21:15:40,999 INFO SenderThread:1974795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval/1_eval_1_ae729244ac49b5c34423.mp4 media/videos/eval/1_eval_1_ae729244ac49b5c34423.mp4 +2025-01-03 21:15:40,999 INFO SenderThread:1974795 [sender.py:transition_state():614] send defer: 10 +2025-01-03 21:15:41,000 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:15:41,000 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:15:41,000 INFO HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-03 21:15:41,000 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: defer +2025-01-03 21:15:41,000 INFO SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-03 21:15:41,000 INFO SenderThread:1974795 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 21:15:41,154 INFO wandb-upload_2:1974795 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/wandb-summary.json +2025-01-03 21:15:41,168 INFO wandb-upload_1:1974795 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/requirements.txt +2025-01-03 21:15:41,178 INFO wandb-upload_0:1974795 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/config.yaml +2025-01-03 21:15:41,223 INFO wandb-upload_3:1974795 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:15:41,424 INFO Thread-11 (_thread_body):1974795 [sender.py:transition_state():614] send defer: 11 +2025-01-03 21:15:41,424 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:15:41,424 INFO HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-03 21:15:41,424 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: defer +2025-01-03 21:15:41,424 INFO SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-03 21:15:41,424 INFO SenderThread:1974795 [file_pusher.py:join():175] waiting for file pusher +2025-01-03 21:15:41,425 INFO SenderThread:1974795 [sender.py:transition_state():614] send defer: 12 +2025-01-03 21:15:41,425 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:15:41,425 INFO HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-03 21:15:41,425 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: defer +2025-01-03 21:15:41,425 INFO SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-03 21:15:41,425 INFO SenderThread:1974795 [file_stream.py:finish():601] file stream finish called +2025-01-03 21:15:41,503 INFO SenderThread:1974795 [file_stream.py:finish():605] file stream finish is done +2025-01-03 21:15:41,504 INFO SenderThread:1974795 [sender.py:transition_state():614] send defer: 13 +2025-01-03 21:15:41,504 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:15:41,504 INFO HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-03 21:15:41,504 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: defer +2025-01-03 21:15:41,504 INFO SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-03 21:15:41,504 INFO SenderThread:1974795 [sender.py:transition_state():614] send defer: 14 +2025-01-03 21:15:41,504 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:15:41,504 INFO HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-03 21:15:41,504 DEBUG SenderThread:1974795 [sender.py:send():379] send: final +2025-01-03 21:15:41,505 DEBUG SenderThread:1974795 [sender.py:send():379] send: footer +2025-01-03 21:15:41,505 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: defer +2025-01-03 21:15:41,505 INFO SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-03 21:15:41,523 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:15:41,523 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:15:41,525 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:41,527 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: get_summary +2025-01-03 21:15:41,527 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-03 21:15:41,529 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: shutdown +2025-01-03 21:15:41,529 INFO HandlerThread:1974795 [handler.py:finish():882] shutting down handler +2025-01-03 21:15:42,523 INFO WriterThread:1974795 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/run-48t61yqi.wandb +2025-01-03 21:15:42,523 INFO SenderThread:1974795 [sender.py:finish():1615] shutting down sender +2025-01-03 21:15:42,523 INFO SenderThread:1974795 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 21:15:42,523 INFO SenderThread:1974795 [file_pusher.py:join():175] waiting for file pusher diff --git a/48t61yqi/wandb/debug.log b/48t61yqi/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f75fa35e0c643221484d49ba445d26a8604fb1be --- /dev/null +++ b/48t61yqi/wandb/debug.log @@ -0,0 +1,37 @@ +2025-01-03 21:13:07,568 INFO MainThread:1974592 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-03 21:13:07,568 INFO MainThread:1974592 [wandb_setup.py:_flush():76] Configure stats pid to 1974592 +2025-01-03 21:13:07,568 INFO MainThread:1974592 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-03 21:13:07,568 INFO MainThread:1974592 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/settings +2025-01-03 21:13:07,568 INFO MainThread:1974592 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-03 21:13:07,568 INFO MainThread:1974592 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-03 21:13:07,568 INFO MainThread:1974592 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-03 21:13:07,568 INFO MainThread:1974592 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-03 21:13:07,568 INFO MainThread:1974592 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/logs/debug.log +2025-01-03 21:13:07,568 INFO MainThread:1974592 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/logs/debug-internal.log +2025-01-03 21:13:07,568 INFO MainThread:1974592 [wandb_init.py:init():569] calling init triggers +2025-01-03 21:13:07,569 INFO MainThread:1974592 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1926_1', 'wandb_notes': '1926_1_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi'} +2025-01-03 21:13:07,569 INFO MainThread:1974592 [wandb_init.py:init():619] starting backend +2025-01-03 21:13:07,569 INFO MainThread:1974592 [wandb_init.py:init():623] setting up manager +2025-01-03 21:13:07,572 INFO MainThread:1974592 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-03 21:13:07,574 INFO MainThread:1974592 [wandb_init.py:init():631] backend started and connected +2025-01-03 21:13:07,586 INFO MainThread:1974592 [wandb_init.py:init():720] updated telemetry +2025-01-03 21:13:07,596 INFO MainThread:1974592 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-03 21:13:07,960 INFO MainThread:1974592 [wandb_run.py:_on_init():2435] communicating current version +2025-01-03 21:13:08,000 INFO MainThread:1974592 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-03 21:13:08,001 INFO MainThread:1974592 [wandb_init.py:init():804] starting run threads in backend +2025-01-03 21:13:08,360 INFO MainThread:1974592 [wandb_run.py:_console_start():2413] atexit reg +2025-01-03 21:13:08,360 INFO MainThread:1974592 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-03 21:13:08,360 INFO MainThread:1974592 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-03 21:13:08,360 INFO MainThread:1974592 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-03 21:13:08,361 INFO MainThread:1974592 [wandb_init.py:init():847] run started, returning control to user process +2025-01-03 21:13:08,362 INFO MainThread:1974592 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/tb, True +2025-01-03 21:13:25,965 INFO MainThread:1974592 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'} +2025-01-03 21:15:29,430 INFO MainThread:1974592 [wandb_run.py:_finish():2107] finishing run serialexperimentsleon/extrinsic_contact_downstream/48t61yqi +2025-01-03 21:15:29,433 INFO MainThread:1974592 [wandb_run.py:_atexit_cleanup():2374] got exitcode: 0 +2025-01-03 21:15:29,434 INFO MainThread:1974592 [wandb_run.py:_restore():2352] restore +2025-01-03 21:15:29,435 INFO MainThread:1974592 [wandb_run.py:_restore():2358] restore done +2025-01-03 21:15:42,533 INFO MainThread:1974592 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-03 21:15:42,537 INFO MainThread:1974592 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-03 21:15:42,608 INFO MainThread:1974592 [wandb_run.py:_footer_sync_info():3975] logging synced files diff --git a/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/code/FISH/eval_robot.py b/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/config.yaml b/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f3984ade38035efcf868f3d55c73cb6107428384 --- /dev/null +++ b/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 1 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1926_1' +wandb_notes: + desc: null + value: 1926_1_restarted_5 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 1 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735956787 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 2 + - 14 + - 16 + - 23 + - 35 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: modelsys +left_book_slot: + desc: null + value: twodim diff --git a/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/diff.patch b/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..c97de4502eea91c4c009a6fb51db681afeec762b --- /dev/null +++ b/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/diff.patch @@ -0,0 +1,57 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 57a851f..9234106 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -138,12 +138,19 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' ++# wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1922_2' # seed 2 ++# wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1006_0' ++wandb_run_id: '1926_1' # seed 1 ++# wandb_run_id: '1930_0' # dataset shuffle seed 1 ++ ++ + + # RGBD+mask+contact(contact+DTC)+act history +-wandb_run_id: '1745_0' ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 71a5d1c..c7eadce 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -159,8 +159,10 @@ run_id_list = [ + ] + + run_id_list = [ +- # '1006_0', +- '1745_0' ++ # '1922_1', ++ # '1922_2', ++ '1926_1', ++ '1930_0', + ] + + checkpoint_epoch = 12000 +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index 6f97e26..285afc8 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,8 +330,8 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.005 +- self.move_to_rot_tolerance = 1.0 ++ self.move_to_trans_tolerance = 0.01 ++ self.move_to_rot_tolerance = 2.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) + self.move_to_target_timeout = 5.0 diff --git a/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval/0_eval_0_ff4d172543858f403e01.mp4 b/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval/0_eval_0_ff4d172543858f403e01.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..18919a3d9962cc192bf35e9e666f393804fce52d --- /dev/null +++ b/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval/0_eval_0_ff4d172543858f403e01.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff4d172543858f403e01d883e2e919457e9b67113e1f51ac9eeb1c12fbd6c7b4 +size 1487478 diff --git a/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval/1_eval_1_ae729244ac49b5c34423.mp4 b/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval/1_eval_1_ae729244ac49b5c34423.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..eba8569934431d34903d1ef46c6912e473ec97ae Binary files /dev/null and b/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval/1_eval_1_ae729244ac49b5c34423.mp4 differ diff --git a/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4 b/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fa868fbb75c51fb59ae42ff30716054804074343 Binary files /dev/null and b/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4 differ diff --git a/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log b/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..2597757529e8f90eb254087697a5767fb1f0d3ba --- /dev/null +++ b/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log @@ -0,0 +1,110 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735956805.969554]: resetting environment +[INFO] [1735956805.977535]: cleared current plan +[INFO] [1735956805.978457]: moving to home +[INFO] [1735956806.980896]: reached home +[INFO] [1735956806.983056]: reset action history +[INFO] [1735956809.692550]: environment reset +Starting episode 0 +[INFO] [1735956809.696665]: resetting environment +[INFO] [1735956809.704118]: cleared current plan +[INFO] [1735956809.705085]: moving to home +[INFO] [1735956810.707855]: reached home +[INFO] [1735956810.709838]: reset action history +[INFO] [1735956813.418834]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1735956816.750033]: resetting environment +[INFO] [1735956816.754664]: cleared current plan +[INFO] [1735956816.755494]: moving to home +[INFO] [1735956817.757227]: reached home +[INFO] [1735956817.759665]: reset action history +[INFO] [1735956820.466643]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/episode_rosbags/episode_0_2025-01-03-21-13-40.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1974592.log +started roslaunch server http://158.130.50.37:46869/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1974904] +process[print_text-2]: started with pid [1974905] +process[rosbag_record-3]: started with pid [1974929] +started bagging! +For topic gripper_width: timestamp difference is 107834998 for nearest: 1735956846261599961 - target: 1735956846153764963 at idx 249 +For topic gripper_width: timestamp difference is 112545745 for nearest: 1735956852028245465 - target: 1735956851915699720 at idx 249 +[INFO] [1735956855.666153]: Storing episode... +[rosbag_record-3] killing on exit +[pub_text-1] killing on exit +[print_text-2] killing on exit +[INFO] [1735956856.438961]: Stored episode 1. +[INFO] [1735956856.439173]: Saving video... +[INFO] [1735956856.639122]: Video saved! +Starting episode 1 +[INFO] [1735956856.654068]: resetting environment +[INFO] [1735956856.663259]: cleared current plan +[INFO] [1735956856.663538]: moving to home +preempting episode +[INFO] [1735956860.366380]: reached home +[INFO] [1735956860.367252]: reset action history +[INFO] [1735956863.078712]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735956896.123101]: resetting environment +[INFO] [1735956896.134941]: cleared current plan +[INFO] [1735956896.135912]: moving to home +[INFO] [1735956897.137384]: reached home +[INFO] [1735956897.138458]: reset action history +[INFO] [1735956899.896548]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/episode_rosbags/episode_1_2025-01-03-21-14-59.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1974592.log +started roslaunch server http://158.130.50.37:39217/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1975115] +process[print_text-5]: started with pid [1975116] +process[rosbag_record-6]: started with pid [1975117] +started bagging! +[INFO] [1735956901.286608]: Storing episode... +[rosbag_record-6] killing on exit[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1735956901.489305]: Stored episode 2. +[INFO] [1735956901.491349]: Saving video... +[INFO] [1735956901.528032]: Video saved! +Starting episode 2 +[INFO] [1735956901.534625]: resetting environment +[INFO] [1735956901.538407]: cleared current plan +[INFO] [1735956901.538596]: moving to home +[INFO] [1735956902.539591]: reached home +[INFO] [1735956902.542495]: reset action history +[INFO] [1735956905.319942]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +program exiting gracefully +proceeding to start episode! +[INFO] [1735956929.387121]: Saving video... +[INFO] [1735956929.394713]: Video saved! \ No newline at end of file diff --git a/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/requirements.txt b/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/wandb-metadata.json b/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..509eed50e4b5b232695112eb2175d3e8d635f24d --- /dev/null +++ b/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-04T02:13:08.025575", + "startedAt": "2025-01-04T02:13:07.566812", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "c537eb6734155592809c2a7db5cca971ec186d72" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3750.002625, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.998, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.061, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 804.7593460083008 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/wandb-summary.json b/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..576e220471c06f16e06905440e4f6f4b202ab37f --- /dev/null +++ b/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "ff4d172543858f403e01d883e2e919457e9b67113e1f51ac9eeb1c12fbd6c7b4", "size": 1487478, "path": "media/videos/eval/0_eval_0_ff4d172543858f403e01.mp4"}, "global_step": 133, "_timestamp": 1735956896.1220462, "_runtime": 108.54749131202698, "_step": 3, "eval/1_eval": {"_type": "video-file", "sha256": "ae729244ac49b5c344235ffc4fd497edb507b67a3c8a88b5fc613756dc78bb18", "size": 65553, "path": "media/videos/eval/1_eval_1_ae729244ac49b5c34423.mp4"}, "eval/video": {"_type": "video-file", "sha256": "25445a41b5bbda0ed7e2e0845eaa5c96373705c5c738fe640c1ea70a07b4176c", "size": 262, "path": "media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4"}, "_wandb": {"runtime": 141}, "eval/num_success": 0.0, "episode": 1.0, "eval/success_rate": 0.0, "eval/success_rate_lb_0.01": 0.0, "eval/success_rate_ub_0.01": 0.9642442464828491, "eval/success_rate_lb_0.025": 0.0, "eval/success_rate_ub_0.025": 0.9641662836074829, "eval/success_rate_lb_0.05": 0.0, "eval/success_rate_ub_0.05": 0.944913387298584, "eval/success_rate_lb_0.1": 0.0, "eval/success_rate_ub_0.1": 0.847197413444519} \ No newline at end of file diff --git a/48t61yqi/wandb/run-20250103_211307-48t61yqi/logs/debug-internal.log b/48t61yqi/wandb/run-20250103_211307-48t61yqi/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..71efbd4cce1a6d839a18232e1bb6e57d660e9471 --- /dev/null +++ b/48t61yqi/wandb/run-20250103_211307-48t61yqi/logs/debug-internal.log @@ -0,0 +1,468 @@ +2025-01-03 21:13:07,574 INFO StreamThr :1974795 [internal.py:wandb_internal():85] W&B internal server running at pid: 1974795, started at: 2025-01-03 21:13:07.573731 +2025-01-03 21:13:07,576 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status +2025-01-03 21:13:07,587 INFO WriterThread:1974795 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/run-48t61yqi.wandb +2025-01-03 21:13:07,587 DEBUG SenderThread:1974795 [sender.py:send():379] send: header +2025-01-03 21:13:07,600 DEBUG SenderThread:1974795 [sender.py:send():379] send: run +2025-01-03 21:13:07,945 INFO SenderThread:1974795 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files +2025-01-03 21:13:07,945 INFO SenderThread:1974795 [sender.py:_start_run_threads():1188] run started: 48t61yqi with start time 1735956787.574555 +2025-01-03 21:13:07,961 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: check_version +2025-01-03 21:13:07,961 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: check_version +2025-01-03 21:13:08,006 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: run_start +2025-01-03 21:13:08,011 DEBUG HandlerThread:1974795 [system_info.py:__init__():26] System info init +2025-01-03 21:13:08,011 DEBUG HandlerThread:1974795 [system_info.py:__init__():41] System info init done +2025-01-03 21:13:08,011 INFO HandlerThread:1974795 [system_monitor.py:start():194] Starting system monitor +2025-01-03 21:13:08,011 INFO SystemMonitor:1974795 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-03 21:13:08,011 INFO HandlerThread:1974795 [system_monitor.py:probe():214] Collecting system info +2025-01-03 21:13:08,012 INFO SystemMonitor:1974795 [interfaces.py:start():188] Started cpu monitoring +2025-01-03 21:13:08,012 INFO SystemMonitor:1974795 [interfaces.py:start():188] Started disk monitoring +2025-01-03 21:13:08,013 INFO SystemMonitor:1974795 [interfaces.py:start():188] Started gpu monitoring +2025-01-03 21:13:08,013 INFO SystemMonitor:1974795 [interfaces.py:start():188] Started memory monitoring +2025-01-03 21:13:08,013 INFO SystemMonitor:1974795 [interfaces.py:start():188] Started network monitoring +2025-01-03 21:13:08,025 DEBUG HandlerThread:1974795 [system_info.py:probe():152] Probing system +2025-01-03 21:13:08,027 DEBUG HandlerThread:1974795 [system_info.py:_probe_git():137] Probing git +2025-01-03 21:13:08,031 DEBUG HandlerThread:1974795 [system_info.py:_probe_git():145] Probing git done +2025-01-03 21:13:08,031 DEBUG HandlerThread:1974795 [system_info.py:probe():200] Probing system done +2025-01-03 21:13:08,031 DEBUG HandlerThread:1974795 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-04T02:13:08.025575', 'startedAt': '2025-01-04T02:13:07.566812', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'c537eb6734155592809c2a7db5cca971ec186d72'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3750.002625, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.998, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.061, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 804.7593460083008}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-03 21:13:08,031 INFO HandlerThread:1974795 [system_monitor.py:probe():224] Finished collecting system info +2025-01-03 21:13:08,031 INFO HandlerThread:1974795 [system_monitor.py:probe():227] Publishing system info +2025-01-03 21:13:08,031 DEBUG HandlerThread:1974795 [system_info.py:_save_code():44] Saving code +2025-01-03 21:13:08,033 DEBUG HandlerThread:1974795 [system_info.py:_save_code():67] Saving code done +2025-01-03 21:13:08,033 DEBUG HandlerThread:1974795 [system_info.py:_save_patches():84] Saving git patches +2025-01-03 21:13:08,094 DEBUG HandlerThread:1974795 [system_info.py:_save_patches():126] Saving git patches done +2025-01-03 21:13:08,095 INFO HandlerThread:1974795 [system_monitor.py:probe():229] Finished publishing system info +2025-01-03 21:13:08,098 DEBUG SenderThread:1974795 [sender.py:send():379] send: files +2025-01-03 21:13:08,098 INFO SenderThread:1974795 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-03 21:13:08,098 INFO SenderThread:1974795 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-03 21:13:08,098 INFO SenderThread:1974795 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-03 21:13:08,340 INFO wandb-upload_2:1974795 [upload_job.py:push():130] Uploaded file /tmp/tmpm2wbqrblwandb/7zaf7m84-diff.patch +2025-01-03 21:13:08,350 INFO wandb-upload_0:1974795 [upload_job.py:push():130] Uploaded file /tmp/tmpm2wbqrblwandb/2v3neepq-wandb-metadata.json +2025-01-03 21:13:08,359 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: python_packages +2025-01-03 21:13:08,360 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: python_packages +2025-01-03 21:13:08,360 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:13:08,360 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:08,361 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:13:08,362 INFO HandlerThread:1974795 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/tb" + save: true +} +control { + relay_id: "106e59ccfedd40bd8337b716b182d46a" +} +_info { + stream_id: "48t61yqi" +} + +2025-01-03 21:13:08,362 DEBUG HandlerThread:1974795 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-03 21:13:08,373 INFO wandb-upload_1:1974795 [upload_job.py:push():130] Uploaded file /tmp/tmpm2wbqrblwandb/wfd4dtnr-code/FISH/eval_robot.py +2025-01-03 21:13:08,515 DEBUG SenderThread:1974795 [sender.py:send():379] send: telemetry +2025-01-03 21:13:08,515 DEBUG SenderThread:1974795 [sender.py:send():379] send: tbrecord +2025-01-03 21:13:08,515 DEBUG SenderThread:1974795 [sender.py:send():379] send: files +2025-01-03 21:13:08,515 INFO SenderThread:1974795 [sender.py:_save_file():1454] saving file events.out.tfevents.1735956788.leonmkim-ROG-Strix-G15CS-G15CS.1974592.0 with policy live +2025-01-03 21:13:08,698 INFO wandb-upload_2:1974795 [upload_job.py:push():130] Uploaded file /tmp/tmpm2wbqrblwandb/utluf6qk-events.out.tfevents.1735956788.leonmkim-ROG-Strix-G15CS-G15CS.1974592.0 +2025-01-03 21:13:08,946 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/requirements.txt +2025-01-03 21:13:08,946 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:13:08,947 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/code/FISH/eval_robot.py +2025-01-03 21:13:08,947 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/wandb-metadata.json +2025-01-03 21:13:08,947 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/events.out.tfevents.1735956788.leonmkim-ROG-Strix-G15CS-G15CS.1974592.0 +2025-01-03 21:13:08,947 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/diff.patch +2025-01-03 21:13:08,947 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/code/FISH +2025-01-03 21:13:08,947 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/code +2025-01-03 21:13:09,360 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:10,360 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:10,945 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:13:11,360 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:12,361 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:13,360 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:13:13,361 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:14,361 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:15,362 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:16,362 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:17,362 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:18,360 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:13:18,362 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:19,362 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:20,362 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:21,362 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:22,362 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:23,360 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:13:23,360 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:13:23,401 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:23,464 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:13:24,362 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:25,363 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:25,966 DEBUG SenderThread:1974795 [sender.py:send():379] send: config +2025-01-03 21:13:26,363 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:26,950 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:13:27,363 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:28,363 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:28,950 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:13:29,360 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:13:29,363 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:30,363 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:30,951 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:13:31,363 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:32,363 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:32,951 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:13:33,363 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:34,364 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:34,461 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:13:34,952 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:13:35,364 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:36,365 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:37,364 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:38,360 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:13:38,360 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:13:38,364 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:38,953 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:13:39,364 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:39,520 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:13:39,953 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/config.yaml +2025-01-03 21:13:40,364 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:41,365 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:42,365 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:42,954 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:13:43,365 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:44,365 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:45,361 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:13:45,365 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:46,365 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:47,365 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:48,366 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:49,365 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:50,361 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:13:50,366 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:51,366 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:52,366 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:53,360 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:13:53,360 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:13:53,401 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:54,366 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:55,366 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:55,497 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:13:56,366 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:57,366 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:58,366 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:13:59,367 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:00,367 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:01,362 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:14:01,367 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:02,367 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:03,367 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:04,367 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:05,367 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:06,368 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:07,313 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:14:07,368 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:08,014 DEBUG SystemMonitor:1974795 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-03 21:14:08,014 DEBUG SenderThread:1974795 [sender.py:send():379] send: stats +2025-01-03 21:14:08,361 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:14:08,361 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:14:08,409 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:08,961 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:14:09,368 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:10,368 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:11,368 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:12,368 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:12,962 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:14:13,073 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:14:13,368 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:14,372 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:15,370 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:16,371 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:16,652 DEBUG SenderThread:1974795 [sender.py:send():379] send: files +2025-01-03 21:14:16,652 INFO SenderThread:1974795 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_ff4d172543858f403e01.mp4 with policy now +2025-01-03 21:14:16,652 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:14:16,653 DEBUG SenderThread:1974795 [sender.py:send():379] send: history +2025-01-03 21:14:16,654 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:14:16,654 INFO SenderThread:1974795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:14:16,878 INFO wandb-upload_1:1974795 [upload_job.py:push():130] Uploaded file /tmp/tmpm2wbqrblwandb/kh7pplhk-media/videos/eval/0_eval_0_ff4d172543858f403e01.mp4 +2025-01-03 21:14:16,963 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:14:16,963 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval/0_eval_0_ff4d172543858f403e01.mp4 +2025-01-03 21:14:16,964 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/wandb-summary.json +2025-01-03 21:14:16,964 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos +2025-01-03 21:14:16,964 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media +2025-01-03 21:14:16,964 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval +2025-01-03 21:14:17,232 INFO wandb-upload_2:1974795 [upload_job.py:push():130] Uploaded file /tmp/tmpm2wbqrblwandb/rsn814pa-media/videos/eval/0_eval_0_ff4d172543858f403e01.mp4 +2025-01-03 21:14:17,371 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:18,362 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:14:18,372 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:18,964 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:14:19,373 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:20,409 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:21,372 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:22,374 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:22,965 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:14:23,361 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:14:23,361 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:14:23,375 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:23,479 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:14:24,374 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:24,966 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:14:25,375 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:26,374 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:27,375 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:28,374 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:29,362 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:14:29,374 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:30,375 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:31,375 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:32,375 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:33,375 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:34,375 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:35,362 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:14:35,375 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:36,376 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:37,376 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:38,015 DEBUG SenderThread:1974795 [sender.py:send():379] send: stats +2025-01-03 21:14:38,361 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:14:38,361 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:14:38,405 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:39,376 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:40,376 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:40,473 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:14:41,377 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:42,377 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:43,377 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:44,378 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:45,377 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:46,363 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:14:46,377 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:47,378 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:48,378 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:49,378 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:50,379 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:51,379 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:52,363 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:14:52,379 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:53,361 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:14:53,362 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:14:53,409 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:54,379 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:55,380 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:56,380 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:56,976 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:14:56,976 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/events.out.tfevents.1735956788.leonmkim-ROG-Strix-G15CS-G15CS.1974592.0 +2025-01-03 21:14:57,223 INFO wandb-upload_1:1974795 [upload_job.py:push():130] Uploaded file /tmp/tmpm2wbqrblwandb/2qynfcuc-events.out.tfevents.1735956788.leonmkim-ROG-Strix-G15CS-G15CS.1974592.0 +2025-01-03 21:14:57,380 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:58,181 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:14:58,380 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:14:58,976 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:14:59,381 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:00,380 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:00,977 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:15:01,380 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:01,533 DEBUG SenderThread:1974795 [sender.py:send():379] send: files +2025-01-03 21:15:01,533 INFO SenderThread:1974795 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_ae729244ac49b5c34423.mp4 with policy now +2025-01-03 21:15:01,533 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:15:01,534 DEBUG SenderThread:1974795 [sender.py:send():379] send: history +2025-01-03 21:15:01,534 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:15:01,535 INFO SenderThread:1974795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:15:01,978 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/wandb-summary.json +2025-01-03 21:15:01,978 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval/1_eval_1_ae729244ac49b5c34423.mp4 +2025-01-03 21:15:01,978 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval +2025-01-03 21:15:02,264 INFO wandb-upload_2:1974795 [upload_job.py:push():130] Uploaded file /tmp/tmpm2wbqrblwandb/ob86mf5s-media/videos/eval/1_eval_1_ae729244ac49b5c34423.mp4 +2025-01-03 21:15:02,389 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:02,513 INFO wandb-upload_1:1974795 [upload_job.py:push():130] Uploaded file /tmp/tmpm2wbqrblwandb/c7fg0d8r-media/videos/eval/1_eval_1_ae729244ac49b5c34423.mp4 +2025-01-03 21:15:02,978 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:15:03,365 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:15:03,389 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:04,384 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:04,979 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:15:05,383 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:06,385 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:06,979 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:15:07,384 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:08,015 DEBUG SenderThread:1974795 [sender.py:send():379] send: stats +2025-01-03 21:15:08,362 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:15:08,363 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:15:08,405 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:08,465 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:15:09,388 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:10,389 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:11,388 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:12,390 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:13,393 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:14,369 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:15:14,390 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:15,393 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:16,391 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:17,396 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:18,396 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:19,372 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:15:19,395 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:20,395 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:21,395 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:22,395 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:23,362 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:15:23,362 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:15:23,409 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:24,397 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:24,467 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:15:25,399 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:26,399 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:27,399 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:28,401 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:29,403 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:29,420 DEBUG SenderThread:1974795 [sender.py:send():379] send: files +2025-01-03 21:15:29,420 INFO SenderThread:1974795 [sender.py:_save_file():1454] saving file media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4 with policy now +2025-01-03 21:15:29,465 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:15:29,465 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: server_info +2025-01-03 21:15:29,466 DEBUG SenderThread:1974795 [sender.py:send():379] send: history +2025-01-03 21:15:29,466 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:15:29,467 INFO SenderThread:1974795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:15:29,468 DEBUG SenderThread:1974795 [sender.py:send():379] send: telemetry +2025-01-03 21:15:29,468 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: server_info +2025-01-03 21:15:29,468 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:15:29,537 DEBUG SenderThread:1974795 [sender.py:send():379] send: exit +2025-01-03 21:15:29,537 INFO SenderThread:1974795 [sender.py:send_exit():586] handling exit code: 0 +2025-01-03 21:15:29,537 INFO SenderThread:1974795 [sender.py:send_exit():588] handling runtime: 141 +2025-01-03 21:15:29,537 INFO SenderThread:1974795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:15:29,537 INFO SenderThread:1974795 [sender.py:send_exit():594] send defer +2025-01-03 21:15:29,537 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:15:29,537 INFO HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-03 21:15:29,538 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: defer +2025-01-03 21:15:29,538 INFO SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-03 21:15:29,538 INFO SenderThread:1974795 [sender.py:transition_state():614] send defer: 1 +2025-01-03 21:15:29,538 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:15:29,538 INFO HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-03 21:15:29,538 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: defer +2025-01-03 21:15:29,538 INFO SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-03 21:15:29,538 INFO SenderThread:1974795 [sender.py:transition_state():614] send defer: 2 +2025-01-03 21:15:29,538 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:15:29,538 INFO HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-03 21:15:29,538 INFO HandlerThread:1974795 [system_monitor.py:finish():203] Stopping system monitor +2025-01-03 21:15:29,538 DEBUG SystemMonitor:1974795 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-03 21:15:29,538 INFO HandlerThread:1974795 [interfaces.py:finish():200] Joined cpu monitor +2025-01-03 21:15:29,538 DEBUG SystemMonitor:1974795 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-03 21:15:29,539 INFO HandlerThread:1974795 [interfaces.py:finish():200] Joined disk monitor +2025-01-03 21:15:29,590 INFO HandlerThread:1974795 [interfaces.py:finish():200] Joined gpu monitor +2025-01-03 21:15:29,591 INFO HandlerThread:1974795 [interfaces.py:finish():200] Joined memory monitor +2025-01-03 21:15:29,591 INFO HandlerThread:1974795 [interfaces.py:finish():200] Joined network monitor +2025-01-03 21:15:29,591 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: defer +2025-01-03 21:15:29,591 INFO SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-03 21:15:29,591 INFO SenderThread:1974795 [sender.py:transition_state():614] send defer: 3 +2025-01-03 21:15:29,591 DEBUG SenderThread:1974795 [sender.py:send():379] send: stats +2025-01-03 21:15:29,591 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:15:29,591 INFO HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-03 21:15:29,592 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: defer +2025-01-03 21:15:29,592 INFO SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-03 21:15:29,592 INFO SenderThread:1974795 [sender.py:transition_state():614] send defer: 4 +2025-01-03 21:15:29,592 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:15:29,592 INFO HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-03 21:15:29,596 INFO wandb-upload_2:1974795 [upload_job.py:push():130] Uploaded file /tmp/tmpm2wbqrblwandb/44ln0e3q-media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4 +2025-01-03 21:15:29,987 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/wandb-summary.json +2025-01-03 21:15:29,987 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4 +2025-01-03 21:15:29,987 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval +2025-01-03 21:15:30,242 INFO wandb-upload_1:1974795 [upload_job.py:push():130] Uploaded file /tmp/tmpm2wbqrblwandb/vh0f04q3-media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4 +2025-01-03 21:15:30,987 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:15:36,759 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:15:36,760 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:15:36,760 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 21:15:36,760 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: defer +2025-01-03 21:15:36,761 INFO SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-03 21:15:36,761 INFO SenderThread:1974795 [sender.py:transition_state():614] send defer: 5 +2025-01-03 21:15:36,761 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:15:36,761 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:15:36,761 INFO HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-03 21:15:36,762 DEBUG SenderThread:1974795 [sender.py:send():379] send: history +2025-01-03 21:15:36,762 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:15:36,762 INFO SenderThread:1974795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:15:36,762 DEBUG SenderThread:1974795 [sender.py:send():379] send: summary +2025-01-03 21:15:36,766 INFO SenderThread:1974795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:15:36,767 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: defer +2025-01-03 21:15:36,767 INFO SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-03 21:15:36,767 INFO SenderThread:1974795 [sender.py:transition_state():614] send defer: 6 +2025-01-03 21:15:36,767 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:15:36,767 INFO HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-03 21:15:36,767 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: defer +2025-01-03 21:15:36,767 INFO SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-03 21:15:36,770 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:15:36,885 INFO SenderThread:1974795 [sender.py:transition_state():614] send defer: 7 +2025-01-03 21:15:36,885 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:15:36,885 INFO HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-03 21:15:36,885 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: defer +2025-01-03 21:15:36,885 INFO SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-03 21:15:36,989 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/config.yaml +2025-01-03 21:15:36,989 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/wandb-summary.json +2025-01-03 21:15:37,544 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:15:38,663 INFO SenderThread:1974795 [sender.py:transition_state():614] send defer: 8 +2025-01-03 21:15:38,664 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:15:38,664 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:15:38,664 INFO HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-03 21:15:38,664 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: defer +2025-01-03 21:15:38,664 INFO SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-03 21:15:38,664 INFO SenderThread:1974795 [job_builder.py:build():440] Attempting to build job artifact +2025-01-03 21:15:38,666 INFO SenderThread:1974795 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-03 21:15:38,680 INFO SenderThread:1974795 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-03 21:15:38,694 INFO SenderThread:1974795 [sender.py:transition_state():614] send defer: 9 +2025-01-03 21:15:38,694 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:15:38,694 INFO HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-03 21:15:38,694 DEBUG SenderThread:1974795 [sender.py:send():379] send: artifact +2025-01-03 21:15:38,990 INFO Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:15:39,322 INFO wandb-upload_0:1974795 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpjy2pywco +2025-01-03 21:15:39,428 INFO wandb-upload_2:1974795 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpqkmuwojg +2025-01-03 21:15:39,482 INFO wandb-upload_1:1974795 [upload_job.py:push():88] Uploaded file /tmp/tmpwr759ffl/wandb-job.json +2025-01-03 21:15:39,558 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:15:40,071 INFO SenderThread:1974795 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNzg5OTM1NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNzg3MjQ4NQ==', 'versionIndex': 408}}} +2025-01-03 21:15:40,072 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: defer +2025-01-03 21:15:40,072 INFO SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-03 21:15:40,072 INFO SenderThread:1974795 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-03 21:15:40,991 INFO SenderThread:1974795 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files +2025-01-03 21:15:40,991 INFO SenderThread:1974795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/config.yaml config.yaml +2025-01-03 21:15:40,991 INFO SenderThread:1974795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/wandb-summary.json wandb-summary.json +2025-01-03 21:15:40,991 INFO SenderThread:1974795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/requirements.txt requirements.txt +2025-01-03 21:15:40,991 INFO SenderThread:1974795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/wandb-metadata.json wandb-metadata.json +2025-01-03 21:15:40,993 INFO SenderThread:1974795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/events.out.tfevents.1735956788.leonmkim-ROG-Strix-G15CS-G15CS.1974592.0 events.out.tfevents.1735956788.leonmkim-ROG-Strix-G15CS-G15CS.1974592.0 +2025-01-03 21:15:40,996 INFO SenderThread:1974795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log output.log +2025-01-03 21:15:40,996 INFO SenderThread:1974795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/diff.patch diff.patch +2025-01-03 21:15:40,999 INFO SenderThread:1974795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-03 21:15:40,999 INFO SenderThread:1974795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4 media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4 +2025-01-03 21:15:40,999 INFO SenderThread:1974795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval/0_eval_0_ff4d172543858f403e01.mp4 media/videos/eval/0_eval_0_ff4d172543858f403e01.mp4 +2025-01-03 21:15:40,999 INFO SenderThread:1974795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval/1_eval_1_ae729244ac49b5c34423.mp4 media/videos/eval/1_eval_1_ae729244ac49b5c34423.mp4 +2025-01-03 21:15:40,999 INFO SenderThread:1974795 [sender.py:transition_state():614] send defer: 10 +2025-01-03 21:15:41,000 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:15:41,000 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:15:41,000 INFO HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-03 21:15:41,000 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: defer +2025-01-03 21:15:41,000 INFO SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-03 21:15:41,000 INFO SenderThread:1974795 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 21:15:41,154 INFO wandb-upload_2:1974795 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/wandb-summary.json +2025-01-03 21:15:41,168 INFO wandb-upload_1:1974795 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/requirements.txt +2025-01-03 21:15:41,178 INFO wandb-upload_0:1974795 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/config.yaml +2025-01-03 21:15:41,223 INFO wandb-upload_3:1974795 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log +2025-01-03 21:15:41,424 INFO Thread-11 (_thread_body):1974795 [sender.py:transition_state():614] send defer: 11 +2025-01-03 21:15:41,424 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:15:41,424 INFO HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-03 21:15:41,424 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: defer +2025-01-03 21:15:41,424 INFO SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-03 21:15:41,424 INFO SenderThread:1974795 [file_pusher.py:join():175] waiting for file pusher +2025-01-03 21:15:41,425 INFO SenderThread:1974795 [sender.py:transition_state():614] send defer: 12 +2025-01-03 21:15:41,425 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:15:41,425 INFO HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-03 21:15:41,425 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: defer +2025-01-03 21:15:41,425 INFO SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-03 21:15:41,425 INFO SenderThread:1974795 [file_stream.py:finish():601] file stream finish called +2025-01-03 21:15:41,503 INFO SenderThread:1974795 [file_stream.py:finish():605] file stream finish is done +2025-01-03 21:15:41,504 INFO SenderThread:1974795 [sender.py:transition_state():614] send defer: 13 +2025-01-03 21:15:41,504 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:15:41,504 INFO HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-03 21:15:41,504 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: defer +2025-01-03 21:15:41,504 INFO SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-03 21:15:41,504 INFO SenderThread:1974795 [sender.py:transition_state():614] send defer: 14 +2025-01-03 21:15:41,504 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:15:41,504 INFO HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-03 21:15:41,504 DEBUG SenderThread:1974795 [sender.py:send():379] send: final +2025-01-03 21:15:41,505 DEBUG SenderThread:1974795 [sender.py:send():379] send: footer +2025-01-03 21:15:41,505 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: defer +2025-01-03 21:15:41,505 INFO SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-03 21:15:41,523 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:15:41,523 DEBUG SenderThread:1974795 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:15:41,525 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:15:41,527 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: get_summary +2025-01-03 21:15:41,527 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-03 21:15:41,529 DEBUG HandlerThread:1974795 [handler.py:handle_request():158] handle_request: shutdown +2025-01-03 21:15:41,529 INFO HandlerThread:1974795 [handler.py:finish():882] shutting down handler +2025-01-03 21:15:42,523 INFO WriterThread:1974795 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/run-48t61yqi.wandb +2025-01-03 21:15:42,523 INFO SenderThread:1974795 [sender.py:finish():1615] shutting down sender +2025-01-03 21:15:42,523 INFO SenderThread:1974795 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 21:15:42,523 INFO SenderThread:1974795 [file_pusher.py:join():175] waiting for file pusher diff --git a/48t61yqi/wandb/run-20250103_211307-48t61yqi/logs/debug.log b/48t61yqi/wandb/run-20250103_211307-48t61yqi/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f75fa35e0c643221484d49ba445d26a8604fb1be --- /dev/null +++ b/48t61yqi/wandb/run-20250103_211307-48t61yqi/logs/debug.log @@ -0,0 +1,37 @@ +2025-01-03 21:13:07,568 INFO MainThread:1974592 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-03 21:13:07,568 INFO MainThread:1974592 [wandb_setup.py:_flush():76] Configure stats pid to 1974592 +2025-01-03 21:13:07,568 INFO MainThread:1974592 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-03 21:13:07,568 INFO MainThread:1974592 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/settings +2025-01-03 21:13:07,568 INFO MainThread:1974592 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-03 21:13:07,568 INFO MainThread:1974592 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-03 21:13:07,568 INFO MainThread:1974592 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-03 21:13:07,568 INFO MainThread:1974592 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-03 21:13:07,568 INFO MainThread:1974592 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/logs/debug.log +2025-01-03 21:13:07,568 INFO MainThread:1974592 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/logs/debug-internal.log +2025-01-03 21:13:07,568 INFO MainThread:1974592 [wandb_init.py:init():569] calling init triggers +2025-01-03 21:13:07,569 INFO MainThread:1974592 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1926_1', 'wandb_notes': '1926_1_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi'} +2025-01-03 21:13:07,569 INFO MainThread:1974592 [wandb_init.py:init():619] starting backend +2025-01-03 21:13:07,569 INFO MainThread:1974592 [wandb_init.py:init():623] setting up manager +2025-01-03 21:13:07,572 INFO MainThread:1974592 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-03 21:13:07,574 INFO MainThread:1974592 [wandb_init.py:init():631] backend started and connected +2025-01-03 21:13:07,586 INFO MainThread:1974592 [wandb_init.py:init():720] updated telemetry +2025-01-03 21:13:07,596 INFO MainThread:1974592 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-03 21:13:07,960 INFO MainThread:1974592 [wandb_run.py:_on_init():2435] communicating current version +2025-01-03 21:13:08,000 INFO MainThread:1974592 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-03 21:13:08,001 INFO MainThread:1974592 [wandb_init.py:init():804] starting run threads in backend +2025-01-03 21:13:08,360 INFO MainThread:1974592 [wandb_run.py:_console_start():2413] atexit reg +2025-01-03 21:13:08,360 INFO MainThread:1974592 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-03 21:13:08,360 INFO MainThread:1974592 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-03 21:13:08,360 INFO MainThread:1974592 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-03 21:13:08,361 INFO MainThread:1974592 [wandb_init.py:init():847] run started, returning control to user process +2025-01-03 21:13:08,362 INFO MainThread:1974592 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/tb, True +2025-01-03 21:13:25,965 INFO MainThread:1974592 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'} +2025-01-03 21:15:29,430 INFO MainThread:1974592 [wandb_run.py:_finish():2107] finishing run serialexperimentsleon/extrinsic_contact_downstream/48t61yqi +2025-01-03 21:15:29,433 INFO MainThread:1974592 [wandb_run.py:_atexit_cleanup():2374] got exitcode: 0 +2025-01-03 21:15:29,434 INFO MainThread:1974592 [wandb_run.py:_restore():2352] restore +2025-01-03 21:15:29,435 INFO MainThread:1974592 [wandb_run.py:_restore():2358] restore done +2025-01-03 21:15:42,533 INFO MainThread:1974592 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-03 21:15:42,537 INFO MainThread:1974592 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-03 21:15:42,608 INFO MainThread:1974592 [wandb_run.py:_footer_sync_info():3975] logging synced files diff --git a/48t61yqi/wandb/run-20250103_211307-48t61yqi/run-48t61yqi.wandb b/48t61yqi/wandb/run-20250103_211307-48t61yqi/run-48t61yqi.wandb new file mode 100644 index 0000000000000000000000000000000000000000..afdcb030b091d7eb4c4b720b3e69c8b045aa8c12 Binary files /dev/null and b/48t61yqi/wandb/run-20250103_211307-48t61yqi/run-48t61yqi.wandb differ diff --git a/dqnz7k8g/eval_policy.log b/dqnz7k8g/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..24590458e817d69d9d44a6b4aad724d54fa04194 --- /dev/null +++ b/dqnz7k8g/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-03 22:52:20,393][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-03 22:52:20,397][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-03 22:52:23,976][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/iqjkgl2e/.hydra/config.yaml b/iqjkgl2e/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5614a531a25d335cfde80ac1396030650a108114 --- /dev/null +++ b/iqjkgl2e/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1926_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/iqjkgl2e/.hydra/hydra.yaml b/iqjkgl2e/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4f8f69d1ebd550b394c753643be73ac223c2097d --- /dev/null +++ b/iqjkgl2e/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/iqjkgl2e/.hydra/overrides.yaml b/iqjkgl2e/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/iqjkgl2e/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/iqjkgl2e/episode_rosbags/aligned_depth_to_color_K.npy b/iqjkgl2e/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/iqjkgl2e/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/iqjkgl2e/episode_rosbags/cam_tf_world.npy b/iqjkgl2e/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/iqjkgl2e/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/iqjkgl2e/episode_rosbags/color_K.npy b/iqjkgl2e/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/iqjkgl2e/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/iqjkgl2e/episode_rosbags/depth_K.npy b/iqjkgl2e/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/iqjkgl2e/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/iqjkgl2e/eval_robot.log b/iqjkgl2e/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..706d31a708c9f417196fb653a8210969b1e7f562 --- /dev/null +++ b/iqjkgl2e/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-04 22:28:33,145][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-04 22:28:33,148][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/iqjkgl2e/eval_video/0_eval.mp4 b/iqjkgl2e/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fc76a00bec747f97d8a70f8fe7894c625d11093b --- /dev/null +++ b/iqjkgl2e/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f829840c9d5231f7c2a1ef7d0f2ce345f32e53845f431d6604f993f0feabd02 +size 1518861 diff --git a/iqjkgl2e/eval_video/1_eval.mp4 b/iqjkgl2e/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..79cfadcb3576011dd9631bd542899deffde14a88 --- /dev/null +++ b/iqjkgl2e/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fcc2c84034b2f80493e2c9bcfdfe89f5533ab66e5acaa9e5a19c94d4ab517e9c +size 1504602 diff --git a/iqjkgl2e/eval_video/2_eval.mp4 b/iqjkgl2e/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9f29cec42243042ccb38777f332c13626b86dc04 --- /dev/null +++ b/iqjkgl2e/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fef06f9228f28f1375655e2e1fa944ee49dc23e690f3c86ce1bc64db99b990e5 +size 1528137 diff --git a/iqjkgl2e/eval_video/3_eval.mp4 b/iqjkgl2e/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..641c294369bca984db8bc5718d9785ba63a5a006 --- /dev/null +++ b/iqjkgl2e/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ceb841f3e55434b9bc762ea3190b2ad5c6886d8ad25c7c25631e6f71967352c +size 1544200 diff --git a/iqjkgl2e/eval_video/4_eval.mp4 b/iqjkgl2e/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..eb83a6c9667d9fc55685dce895dd6536e1d184c5 --- /dev/null +++ b/iqjkgl2e/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d691d8c6f8564711e47c11a3353dd90ba7d94653918a5c63f47690e8a8b9c0ae +size 1542124 diff --git a/iqjkgl2e/tb/events.out.tfevents.1736047719.leonmkim-ROG-Strix-G15CS-G15CS.2037421.0 b/iqjkgl2e/tb/events.out.tfevents.1736047719.leonmkim-ROG-Strix-G15CS-G15CS.2037421.0 new file mode 100644 index 0000000000000000000000000000000000000000..24daf5af9c0919a26f834a79d26ae633aa166701 --- /dev/null +++ b/iqjkgl2e/tb/events.out.tfevents.1736047719.leonmkim-ROG-Strix-G15CS-G15CS.2037421.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21bf7564b93ce583dfb9f81a429595fbae0b4760f63c86d77529d4cb089ca31b +size 6539 diff --git a/iqjkgl2e/wandb/debug-internal.log b/iqjkgl2e/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..999ec1f7138dc7232f27cce471baf7750f87bad9 --- /dev/null +++ b/iqjkgl2e/wandb/debug-internal.log @@ -0,0 +1,886 @@ +2025-01-04 22:28:38,823 INFO StreamThr :2037873 [internal.py:wandb_internal():85] W&B internal server running at pid: 2037873, started at: 2025-01-04 22:28:38.822989 +2025-01-04 22:28:38,824 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status +2025-01-04 22:28:38,835 INFO WriterThread:2037873 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/run-iqjkgl2e.wandb +2025-01-04 22:28:38,835 DEBUG SenderThread:2037873 [sender.py:send():379] send: header +2025-01-04 22:28:38,845 DEBUG SenderThread:2037873 [sender.py:send():379] send: run +2025-01-04 22:28:39,074 INFO SenderThread:2037873 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files +2025-01-04 22:28:39,074 INFO SenderThread:2037873 [sender.py:_start_run_threads():1188] run started: iqjkgl2e with start time 1736047718.823026 +2025-01-04 22:28:39,090 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: check_version +2025-01-04 22:28:39,090 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: check_version +2025-01-04 22:28:39,138 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: run_start +2025-01-04 22:28:39,143 DEBUG HandlerThread:2037873 [system_info.py:__init__():26] System info init +2025-01-04 22:28:39,143 DEBUG HandlerThread:2037873 [system_info.py:__init__():41] System info init done +2025-01-04 22:28:39,143 INFO HandlerThread:2037873 [system_monitor.py:start():194] Starting system monitor +2025-01-04 22:28:39,143 INFO SystemMonitor:2037873 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-04 22:28:39,143 INFO HandlerThread:2037873 [system_monitor.py:probe():214] Collecting system info +2025-01-04 22:28:39,144 INFO SystemMonitor:2037873 [interfaces.py:start():188] Started cpu monitoring +2025-01-04 22:28:39,144 INFO SystemMonitor:2037873 [interfaces.py:start():188] Started disk monitoring +2025-01-04 22:28:39,145 INFO SystemMonitor:2037873 [interfaces.py:start():188] Started gpu monitoring +2025-01-04 22:28:39,145 INFO SystemMonitor:2037873 [interfaces.py:start():188] Started memory monitoring +2025-01-04 22:28:39,145 INFO SystemMonitor:2037873 [interfaces.py:start():188] Started network monitoring +2025-01-04 22:28:39,155 DEBUG HandlerThread:2037873 [system_info.py:probe():152] Probing system +2025-01-04 22:28:39,156 DEBUG HandlerThread:2037873 [system_info.py:_probe_git():137] Probing git +2025-01-04 22:28:39,161 DEBUG HandlerThread:2037873 [system_info.py:_probe_git():145] Probing git done +2025-01-04 22:28:39,161 DEBUG HandlerThread:2037873 [system_info.py:probe():200] Probing system done +2025-01-04 22:28:39,161 DEBUG HandlerThread:2037873 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-05T03:28:39.155554', 'startedAt': '2025-01-05T03:28:38.811012', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '02cf07d901f3d8187a302cd51c9faa80321417ee'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3560.90725, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4495.482, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4493.257, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 762.5235061645508}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-04 22:28:39,161 INFO HandlerThread:2037873 [system_monitor.py:probe():224] Finished collecting system info +2025-01-04 22:28:39,161 INFO HandlerThread:2037873 [system_monitor.py:probe():227] Publishing system info +2025-01-04 22:28:39,161 DEBUG HandlerThread:2037873 [system_info.py:_save_code():44] Saving code +2025-01-04 22:28:39,168 DEBUG HandlerThread:2037873 [system_info.py:_save_code():67] Saving code done +2025-01-04 22:28:39,168 DEBUG HandlerThread:2037873 [system_info.py:_save_patches():84] Saving git patches +2025-01-04 22:28:39,233 DEBUG HandlerThread:2037873 [system_info.py:_save_patches():126] Saving git patches done +2025-01-04 22:28:39,233 INFO HandlerThread:2037873 [system_monitor.py:probe():229] Finished publishing system info +2025-01-04 22:28:39,237 DEBUG SenderThread:2037873 [sender.py:send():379] send: files +2025-01-04 22:28:39,237 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-04 22:28:39,237 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-04 22:28:39,238 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-04 22:28:39,488 INFO wandb-upload_0:2037873 [upload_job.py:push():130] Uploaded file /tmp/tmpn9pnfdm3wandb/poa4dgwj-wandb-metadata.json +2025-01-04 22:28:39,488 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: python_packages +2025-01-04 22:28:39,488 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: python_packages +2025-01-04 22:28:39,488 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:39,489 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:28:39,489 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:28:39,504 INFO wandb-upload_2:2037873 [upload_job.py:push():130] Uploaded file /tmp/tmpn9pnfdm3wandb/mm48v6aw-diff.patch +2025-01-04 22:28:39,533 INFO HandlerThread:2037873 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/tb" + save: true +} +control { + relay_id: "b649c796fe424680bb0ddd3c2c946087" +} +_info { + stream_id: "iqjkgl2e" +} + +2025-01-04 22:28:39,533 DEBUG HandlerThread:2037873 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-04 22:28:39,537 INFO wandb-upload_1:2037873 [upload_job.py:push():130] Uploaded file /tmp/tmpn9pnfdm3wandb/b6rj74pm-code/FISH/eval_robot.py +2025-01-04 22:28:39,617 DEBUG SenderThread:2037873 [sender.py:send():379] send: telemetry +2025-01-04 22:28:39,617 DEBUG SenderThread:2037873 [sender.py:send():379] send: tbrecord +2025-01-04 22:28:39,618 DEBUG SenderThread:2037873 [sender.py:send():379] send: files +2025-01-04 22:28:39,618 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file events.out.tfevents.1736047719.leonmkim-ROG-Strix-G15CS-G15CS.2037421.0 with policy live +2025-01-04 22:28:39,804 INFO wandb-upload_0:2037873 [upload_job.py:push():130] Uploaded file /tmp/tmpn9pnfdm3wandb/ryljuk5s-events.out.tfevents.1736047719.leonmkim-ROG-Strix-G15CS-G15CS.2037421.0 +2025-01-04 22:28:40,076 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:28:40,076 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/wandb-metadata.json +2025-01-04 22:28:40,076 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/diff.patch +2025-01-04 22:28:40,076 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/requirements.txt +2025-01-04 22:28:40,077 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/events.out.tfevents.1736047719.leonmkim-ROG-Strix-G15CS-G15CS.2037421.0 +2025-01-04 22:28:40,077 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/code/FISH/eval_robot.py +2025-01-04 22:28:40,077 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/code +2025-01-04 22:28:40,077 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/code/FISH +2025-01-04 22:28:40,488 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:41,488 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:42,075 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:28:42,488 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:43,488 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:44,488 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:28:44,489 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:45,489 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:46,489 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:47,489 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:48,489 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:49,488 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:28:49,489 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:50,489 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:51,489 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:52,490 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:52,930 DEBUG SenderThread:2037873 [sender.py:send():379] send: config +2025-01-04 22:28:53,490 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:54,078 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:28:54,488 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:28:54,488 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:28:54,490 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:54,620 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:28:55,490 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:56,079 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:28:56,490 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:57,490 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:58,080 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:28:58,490 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:59,490 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:00,081 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:29:00,372 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:29:00,490 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:01,490 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:02,082 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:29:02,491 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:03,491 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:04,491 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:05,489 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:29:05,491 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:06,491 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:07,491 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:08,491 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:09,488 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:29:09,488 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:29:09,529 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:10,491 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:10,647 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:29:11,084 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/config.yaml +2025-01-04 22:29:11,491 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:12,492 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:13,492 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:14,492 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:15,493 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:16,489 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:29:16,493 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:17,493 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:18,493 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:19,493 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:20,493 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:21,490 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:29:21,493 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:22,493 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:23,494 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:24,488 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:29:24,488 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:29:24,533 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:25,494 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:26,494 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:26,596 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:29:27,494 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:28,089 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:29:28,494 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:29,494 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:30,090 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:29:30,529 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:31,495 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:31,779 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:29:32,090 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:29:32,495 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:33,495 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:34,495 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:35,495 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:36,495 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:37,490 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:29:37,495 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:38,495 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:39,146 DEBUG SystemMonitor:2037873 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-04 22:29:39,147 DEBUG SenderThread:2037873 [sender.py:send():379] send: stats +2025-01-04 22:29:39,488 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:29:39,489 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:29:39,533 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:40,495 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:41,496 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:42,496 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:42,580 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:29:43,496 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:44,496 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:45,496 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:46,497 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:47,497 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:48,490 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:29:48,497 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:49,497 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:50,497 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:51,500 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:52,096 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:29:52,498 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:53,498 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:53,685 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:29:54,096 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:29:54,488 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:29:54,489 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:29:54,529 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:55,498 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:56,097 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:29:56,498 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:57,498 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:58,499 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:58,758 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:29:59,499 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:00,499 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:01,499 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:02,499 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:03,500 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:04,490 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:30:04,499 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:05,499 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:06,286 DEBUG SenderThread:2037873 [sender.py:send():379] send: files +2025-01-04 22:30:06,286 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_3f829840c9d5231f7c2a.mp4 with policy now +2025-01-04 22:30:06,286 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:30:06,287 DEBUG SenderThread:2037873 [sender.py:send():379] send: history +2025-01-04 22:30:06,287 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:30:06,288 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:30:06,501 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:06,537 INFO wandb-upload_1:2037873 [upload_job.py:push():130] Uploaded file /tmp/tmpn9pnfdm3wandb/2gejalib-media/videos/eval/0_eval_0_3f829840c9d5231f7c2a.mp4 +2025-01-04 22:30:07,100 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval/0_eval_0_3f829840c9d5231f7c2a.mp4 +2025-01-04 22:30:07,100 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/wandb-summary.json +2025-01-04 22:30:07,100 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval +2025-01-04 22:30:07,100 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media +2025-01-04 22:30:07,100 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos +2025-01-04 22:30:07,388 INFO wandb-upload_0:2037873 [upload_job.py:push():130] Uploaded file /tmp/tmpn9pnfdm3wandb/ewfh9atc-media/videos/eval/0_eval_0_3f829840c9d5231f7c2a.mp4 +2025-01-04 22:30:07,500 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:08,100 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:30:08,500 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:09,147 DEBUG SenderThread:2037873 [sender.py:send():379] send: stats +2025-01-04 22:30:09,489 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:30:09,489 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:30:09,529 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:09,608 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:30:10,500 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:11,500 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:12,101 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:30:12,500 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:13,500 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:14,102 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:30:14,501 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:15,491 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:30:15,501 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:16,501 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:17,501 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:18,501 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:19,501 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:20,491 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:30:20,501 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:21,501 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:22,501 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:23,502 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:24,489 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:30:24,489 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:30:24,529 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:25,502 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:25,619 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:30:26,502 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:27,502 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:28,503 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:29,503 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:30,107 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/events.out.tfevents.1736047719.leonmkim-ROG-Strix-G15CS-G15CS.2037421.0 +2025-01-04 22:30:30,347 INFO wandb-upload_1:2037873 [upload_job.py:push():130] Uploaded file /tmp/tmpn9pnfdm3wandb/mqkj06tu-events.out.tfevents.1736047719.leonmkim-ROG-Strix-G15CS-G15CS.2037421.0 +2025-01-04 22:30:30,503 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:31,503 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:31,577 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:30:32,108 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:30:32,503 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:33,503 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:34,504 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:35,504 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:36,110 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:30:36,504 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:37,491 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:30:37,504 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:38,504 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:39,154 DEBUG SenderThread:2037873 [sender.py:send():379] send: stats +2025-01-04 22:30:39,489 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:30:39,489 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:30:39,529 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:40,505 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:41,505 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:42,505 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:43,169 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:30:43,505 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:44,112 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:30:44,505 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:45,505 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:46,113 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:30:46,505 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:47,505 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:48,492 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:30:48,505 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:49,505 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:50,114 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:30:50,506 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:51,506 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:52,506 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:53,492 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:30:53,506 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:54,489 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:30:54,489 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:30:54,506 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:55,506 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:56,116 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:30:56,506 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:57,506 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:58,507 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:58,609 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:30:59,506 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:00,507 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:01,508 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:02,508 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:03,508 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:04,492 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:31:04,508 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:05,508 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:06,508 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:07,508 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:08,509 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:09,148 DEBUG SenderThread:2037873 [sender.py:send():379] send: stats +2025-01-04 22:31:09,495 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:31:09,498 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:31:09,498 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:31:09,518 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:09,561 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:31:09,562 DEBUG SenderThread:2037873 [sender.py:send():379] send: files +2025-01-04 22:31:09,562 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_fcc2c84034b2f80493e2.mp4 with policy now +2025-01-04 22:31:09,563 DEBUG SenderThread:2037873 [sender.py:send():379] send: history +2025-01-04 22:31:09,563 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:31:09,568 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:31:09,814 INFO wandb-upload_0:2037873 [upload_job.py:push():130] Uploaded file /tmp/tmpn9pnfdm3wandb/0o5eybqx-media/videos/eval/1_eval_1_fcc2c84034b2f80493e2.mp4 +2025-01-04 22:31:10,120 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:31:10,120 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/wandb-summary.json +2025-01-04 22:31:10,121 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval/1_eval_1_fcc2c84034b2f80493e2.mp4 +2025-01-04 22:31:10,121 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval +2025-01-04 22:31:10,381 INFO wandb-upload_1:2037873 [upload_job.py:push():130] Uploaded file /tmp/tmpn9pnfdm3wandb/b18t9tm8-media/videos/eval/1_eval_1_fcc2c84034b2f80493e2.mp4 +2025-01-04 22:31:10,510 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:11,510 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:12,121 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:31:12,510 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:13,510 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:14,122 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:31:14,499 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:31:14,514 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:15,512 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:16,510 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:17,511 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:18,123 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:31:18,511 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:19,511 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:20,495 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:31:20,512 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:21,511 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:22,512 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:23,512 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:24,491 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:31:24,491 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:31:24,537 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:25,513 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:25,588 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:31:26,513 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:27,000 DEBUG SenderThread:2037873 [sender.py:send():379] send: history +2025-01-04 22:31:27,000 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:31:27,001 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:31:27,126 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/wandb-summary.json +2025-01-04 22:31:27,127 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/events.out.tfevents.1736047719.leonmkim-ROG-Strix-G15CS-G15CS.2037421.0 +2025-01-04 22:31:27,512 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:28,126 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:31:28,512 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:29,513 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:30,513 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:31,193 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:31:31,513 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:32,128 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:31:32,513 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:33,513 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:34,513 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:35,513 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:36,496 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:31:36,513 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:37,513 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:38,513 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:39,149 DEBUG SenderThread:2037873 [sender.py:send():379] send: stats +2025-01-04 22:31:39,491 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:31:39,491 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:31:39,533 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:40,514 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:41,514 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:41,601 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:31:42,514 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:43,514 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:44,514 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:45,516 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:46,515 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:47,496 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:31:47,515 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:48,516 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:49,516 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:50,516 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:51,516 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:52,496 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:31:52,516 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:53,516 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:54,491 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:31:54,491 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:31:54,533 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:55,516 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:56,516 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:57,516 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:57,565 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:31:58,517 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:59,517 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:00,137 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:32:00,518 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:01,518 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:02,138 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:32:02,518 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:03,429 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:32:03,518 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:04,518 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:05,522 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:06,139 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:32:06,336 DEBUG SenderThread:2037873 [sender.py:send():379] send: files +2025-01-04 22:32:06,336 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_fef06f9228f28f137565.mp4 with policy now +2025-01-04 22:32:06,339 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:32:06,339 DEBUG SenderThread:2037873 [sender.py:send():379] send: history +2025-01-04 22:32:06,339 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:32:06,347 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:32:06,519 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:06,598 INFO wandb-upload_0:2037873 [upload_job.py:push():130] Uploaded file /tmp/tmpn9pnfdm3wandb/lkqj9xbf-media/videos/eval/2_eval_2_fef06f9228f28f137565.mp4 +2025-01-04 22:32:07,139 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/wandb-summary.json +2025-01-04 22:32:07,139 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval/2_eval_2_fef06f9228f28f137565.mp4 +2025-01-04 22:32:07,140 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval +2025-01-04 22:32:07,449 INFO wandb-upload_1:2037873 [upload_job.py:push():130] Uploaded file /tmp/tmpn9pnfdm3wandb/l3mqs4sy-media/videos/eval/2_eval_2_fef06f9228f28f137565.mp4 +2025-01-04 22:32:07,522 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:08,140 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:32:08,524 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:09,149 DEBUG SenderThread:2037873 [sender.py:send():379] send: stats +2025-01-04 22:32:09,149 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:32:09,493 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:32:09,493 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:32:09,537 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:10,525 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:11,532 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:12,141 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:32:12,549 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:13,523 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:14,142 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:32:14,497 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:32:14,525 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:15,525 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:16,527 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:17,533 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:18,527 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:19,498 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:32:19,537 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:20,527 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:21,528 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:22,532 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:23,534 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:24,496 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:32:24,496 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:32:24,537 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:24,581 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:32:25,530 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:26,533 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:27,535 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:28,146 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/events.out.tfevents.1736047719.leonmkim-ROG-Strix-G15CS-G15CS.2037421.0 +2025-01-04 22:32:28,385 INFO wandb-upload_0:2037873 [upload_job.py:push():130] Uploaded file /tmp/tmpn9pnfdm3wandb/cfupijsq-events.out.tfevents.1736047719.leonmkim-ROG-Strix-G15CS-G15CS.2037421.0 +2025-01-04 22:32:28,538 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:28,858 DEBUG SenderThread:2037873 [sender.py:send():379] send: history +2025-01-04 22:32:28,858 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:32:28,858 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:32:29,146 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/wandb-summary.json +2025-01-04 22:32:29,539 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:30,147 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:32:30,499 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:32:30,538 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:31,539 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:32,537 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:33,537 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:34,148 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:32:34,537 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:35,537 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:35,773 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:32:36,149 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:32:36,537 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:37,537 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:38,150 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:32:38,537 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:39,149 DEBUG SenderThread:2037873 [sender.py:send():379] send: stats +2025-01-04 22:32:39,493 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:32:39,493 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:32:39,538 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:40,150 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:32:40,538 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:41,538 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:41,621 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:32:42,538 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:43,538 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:44,538 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:45,538 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:46,153 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:32:46,539 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:47,500 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:32:47,539 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:48,539 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:49,539 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:50,539 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:51,539 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:52,500 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:32:52,540 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:53,540 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:54,493 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:32:54,494 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:32:54,540 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:55,540 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:56,540 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:57,540 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:57,566 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:32:58,540 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:59,540 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:00,540 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:01,541 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:02,541 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:03,500 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:33:03,543 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:04,541 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:05,541 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:06,542 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:07,549 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:08,160 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:33:08,374 DEBUG SenderThread:2037873 [sender.py:send():379] send: files +2025-01-04 22:33:08,374 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_2ceb841f3e55434b9bc7.mp4 with policy now +2025-01-04 22:33:08,378 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:33:08,379 DEBUG SenderThread:2037873 [sender.py:send():379] send: history +2025-01-04 22:33:08,379 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:33:08,381 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:33:08,549 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:08,621 INFO wandb-upload_1:2037873 [upload_job.py:push():130] Uploaded file /tmp/tmpn9pnfdm3wandb/lhgnl3iz-media/videos/eval/3_eval_3_2ceb841f3e55434b9bc7.mp4 +2025-01-04 22:33:09,150 DEBUG SenderThread:2037873 [sender.py:send():379] send: stats +2025-01-04 22:33:09,150 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:33:09,160 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/wandb-summary.json +2025-01-04 22:33:09,160 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval/3_eval_3_2ceb841f3e55434b9bc7.mp4 +2025-01-04 22:33:09,160 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval +2025-01-04 22:33:09,445 INFO wandb-upload_0:2037873 [upload_job.py:push():130] Uploaded file /tmp/tmpn9pnfdm3wandb/rezj31aj-media/videos/eval/3_eval_3_2ceb841f3e55434b9bc7.mp4 +2025-01-04 22:33:09,494 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:33:09,494 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:33:09,554 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:10,160 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:33:10,563 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:11,554 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:12,555 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:13,557 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:14,162 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:33:14,504 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:33:14,562 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:15,560 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:16,162 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:33:16,563 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:17,558 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:18,163 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/events.out.tfevents.1736047719.leonmkim-ROG-Strix-G15CS-G15CS.2037421.0 +2025-01-04 22:33:18,562 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:18,919 DEBUG SenderThread:2037873 [sender.py:send():379] send: history +2025-01-04 22:33:18,919 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:33:18,920 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:33:19,164 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/wandb-summary.json +2025-01-04 22:33:19,563 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:20,164 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:33:20,502 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:33:20,567 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:21,563 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:22,562 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:23,562 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:24,165 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:33:24,494 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:33:24,494 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:33:24,562 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:25,563 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:25,865 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:33:26,166 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:33:26,563 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:27,563 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:28,563 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:29,563 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:30,563 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:31,502 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:33:31,564 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:32,564 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:33,564 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:34,564 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:35,564 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:36,503 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:33:36,565 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:37,565 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:38,565 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:39,156 DEBUG SenderThread:2037873 [sender.py:send():379] send: stats +2025-01-04 22:33:39,494 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:33:39,494 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:33:39,565 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:40,565 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:41,565 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:33:41,565 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:42,565 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:43,565 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:44,566 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:45,566 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:46,566 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:47,502 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:33:47,566 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:48,566 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:49,566 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:50,566 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:51,566 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:52,503 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:33:52,566 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:53,566 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:54,494 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:33:54,494 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:33:54,567 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:55,567 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:56,567 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:57,574 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:57,670 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:33:58,172 DEBUG SenderThread:2037873 [sender.py:send():379] send: files +2025-01-04 22:33:58,172 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_d691d8c6f8564711e47c.mp4 with policy now +2025-01-04 22:33:58,173 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:33:58,173 DEBUG SenderThread:2037873 [sender.py:send():379] send: history +2025-01-04 22:33:58,173 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:33:58,177 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:33:58,177 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/wandb-summary.json +2025-01-04 22:33:58,177 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval/4_eval_4_d691d8c6f8564711e47c.mp4 +2025-01-04 22:33:58,177 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval +2025-01-04 22:33:58,187 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:33:58,430 INFO wandb-upload_1:2037873 [upload_job.py:push():130] Uploaded file /tmp/tmpn9pnfdm3wandb/k0tg9pjp-media/videos/eval/4_eval_4_d691d8c6f8564711e47c.mp4 +2025-01-04 22:33:58,568 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:58,724 INFO wandb-upload_0:2037873 [upload_job.py:push():130] Uploaded file /tmp/tmpn9pnfdm3wandb/0l4m7nbc-media/videos/eval/4_eval_4_d691d8c6f8564711e47c.mp4 +2025-01-04 22:33:59,587 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:00,178 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:34:00,574 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:01,581 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:02,579 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:02,769 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:34:03,584 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:04,179 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:34:04,588 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:05,590 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:06,180 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:34:06,590 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:07,587 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:08,557 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:34:08,598 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:09,151 DEBUG SenderThread:2037873 [sender.py:send():379] send: stats +2025-01-04 22:34:09,497 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:34:09,497 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:34:09,593 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:10,595 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:10,946 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 22:34:10,946 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 22:34:11,141 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:34:11,182 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/events.out.tfevents.1736047719.leonmkim-ROG-Strix-G15CS-G15CS.2037421.0 +2025-01-04 22:34:11,182 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/table/eval/success_rate_ci_5_2d447e7da5390d365153.table.json +2025-01-04 22:34:11,182 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media +2025-01-04 22:34:11,182 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/table/eval +2025-01-04 22:34:11,182 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/table +2025-01-04 22:34:11,394 INFO wandb-upload_0:2037873 [upload_job.py:push():130] Uploaded file /tmp/tmpn9pnfdm3wandb/qnz3jgu8-media/table/eval/success_rate_ci_5_2d447e7da5390d365153.table.json +2025-01-04 22:34:11,403 INFO wandb-upload_1:2037873 [upload_job.py:push():130] Uploaded file /tmp/tmpn9pnfdm3wandb/oe9jyqc9-events.out.tfevents.1736047719.leonmkim-ROG-Strix-G15CS-G15CS.2037421.0 +2025-01-04 22:34:11,434 INFO wandb-upload_2:2037873 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmprs_byj3h +2025-01-04 22:34:11,574 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 22:34:11,584 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:34:11,592 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:11,830 INFO SenderThread:2037873 [sender.py:send_request_log_artifact():1525] logged artifact run-iqjkgl2e-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE5NTkzNw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg2MzM5Mg==', 'latestArtifact': None}} +2025-01-04 22:34:11,830 DEBUG SenderThread:2037873 [sender.py:send():379] send: history +2025-01-04 22:34:11,830 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:34:11,831 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:34:11,831 DEBUG SenderThread:2037873 [sender.py:send():379] send: files +2025-01-04 22:34:11,831 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_2d447e7da5390d365153.table.json with policy now +2025-01-04 22:34:11,831 DEBUG SenderThread:2037873 [sender.py:send():379] send: history +2025-01-04 22:34:11,831 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:34:11,831 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:34:11,832 DEBUG SenderThread:2037873 [sender.py:send():379] send: telemetry +2025-01-04 22:34:11,832 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:34:11,832 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:34:11,832 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:34:11,833 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:34:11,833 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:34:11,833 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:34:11,833 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:34:11,834 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:34:11,834 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:34:11,834 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:34:11,834 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:34:11,835 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:34:11,835 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:34:11,837 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:34:11,837 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:34:11,837 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:34:11,837 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:34:11,838 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:34:11,838 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:34:11,838 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:34:11,838 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:34:11,839 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:34:11,839 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 22:34:12,071 INFO wandb-upload_0:2037873 [upload_job.py:push():130] Uploaded file /tmp/tmpn9pnfdm3wandb/7pij9etm-media/table/eval/success_rate_ci_5_2d447e7da5390d365153.table.json +2025-01-04 22:34:12,182 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:34:12,182 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/wandb-summary.json +2025-01-04 22:34:12,182 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/events.out.tfevents.1736047719.leonmkim-ROG-Strix-G15CS-G15CS.2037421.0 +2025-01-04 22:34:12,182 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/table/eval/total_success_rate_ci_6_157933031db0c923cb36.table.json +2025-01-04 22:34:12,183 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/table/eval +2025-01-04 22:34:12,192 INFO wandb-upload_2:2037873 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpx0hyb_gx +2025-01-04 22:34:12,384 INFO wandb-upload_1:2037873 [upload_job.py:push():130] Uploaded file /tmp/tmpn9pnfdm3wandb/qhw5avsz-media/table/eval/total_success_rate_ci_6_157933031db0c923cb36.table.json +2025-01-04 22:34:12,591 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:12,605 INFO SenderThread:2037873 [sender.py:send_request_log_artifact():1525] logged artifact run-iqjkgl2e-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE5NTk0Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg2MzM5NA==', 'latestArtifact': None}} +2025-01-04 22:34:12,605 DEBUG SenderThread:2037873 [sender.py:send():379] send: files +2025-01-04 22:34:12,605 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_157933031db0c923cb36.table.json with policy now +2025-01-04 22:34:12,605 DEBUG SenderThread:2037873 [sender.py:send():379] send: history +2025-01-04 22:34:12,605 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:34:12,606 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:34:12,832 INFO wandb-upload_0:2037873 [upload_job.py:push():130] Uploaded file /tmp/tmpn9pnfdm3wandb/n4ysvd5d-media/table/eval/total_success_rate_ci_6_157933031db0c923cb36.table.json +2025-01-04 22:34:13,182 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/wandb-summary.json +2025-01-04 22:34:13,591 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:13,606 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:34:14,183 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:34:14,591 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:15,592 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:16,592 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:17,592 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:18,592 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:19,506 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:34:19,592 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:20,592 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:21,592 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:22,592 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:23,593 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:24,494 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:34:24,494 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:34:24,568 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:34:24,593 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:25,593 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:26,593 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:27,593 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:28,206 DEBUG SenderThread:2037873 [sender.py:send():379] send: exit +2025-01-04 22:34:28,206 INFO SenderThread:2037873 [sender.py:send_exit():586] handling exit code: 0 +2025-01-04 22:34:28,206 INFO SenderThread:2037873 [sender.py:send_exit():588] handling runtime: 349 +2025-01-04 22:34:28,206 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:34:28,207 INFO SenderThread:2037873 [sender.py:send_exit():594] send defer +2025-01-04 22:34:28,207 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:34:28,207 INFO HandlerThread:2037873 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-04 22:34:28,207 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: defer +2025-01-04 22:34:28,207 INFO SenderThread:2037873 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-04 22:34:28,207 INFO SenderThread:2037873 [sender.py:transition_state():614] send defer: 1 +2025-01-04 22:34:28,207 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:34:28,207 INFO HandlerThread:2037873 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-04 22:34:28,207 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: defer +2025-01-04 22:34:28,207 INFO SenderThread:2037873 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-04 22:34:28,207 INFO SenderThread:2037873 [sender.py:transition_state():614] send defer: 2 +2025-01-04 22:34:28,208 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:34:28,208 INFO HandlerThread:2037873 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-04 22:34:28,208 INFO HandlerThread:2037873 [system_monitor.py:finish():203] Stopping system monitor +2025-01-04 22:34:28,208 DEBUG SystemMonitor:2037873 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-04 22:34:28,208 DEBUG SystemMonitor:2037873 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-04 22:34:28,209 INFO HandlerThread:2037873 [interfaces.py:finish():200] Joined cpu monitor +2025-01-04 22:34:28,209 INFO HandlerThread:2037873 [interfaces.py:finish():200] Joined disk monitor +2025-01-04 22:34:28,261 INFO HandlerThread:2037873 [interfaces.py:finish():200] Joined gpu monitor +2025-01-04 22:34:28,262 INFO HandlerThread:2037873 [interfaces.py:finish():200] Joined memory monitor +2025-01-04 22:34:28,262 INFO HandlerThread:2037873 [interfaces.py:finish():200] Joined network monitor +2025-01-04 22:34:28,262 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: defer +2025-01-04 22:34:28,262 INFO SenderThread:2037873 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-04 22:34:28,262 INFO SenderThread:2037873 [sender.py:transition_state():614] send defer: 3 +2025-01-04 22:34:28,262 DEBUG SenderThread:2037873 [sender.py:send():379] send: stats +2025-01-04 22:34:28,263 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:34:28,263 INFO HandlerThread:2037873 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-04 22:34:28,263 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: defer +2025-01-04 22:34:28,263 INFO SenderThread:2037873 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-04 22:34:28,263 INFO SenderThread:2037873 [sender.py:transition_state():614] send defer: 4 +2025-01-04 22:34:28,263 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:34:28,263 INFO HandlerThread:2037873 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-04 22:34:29,189 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/wandb-summary.json +2025-01-04 22:34:30,190 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:34:34,988 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:34:34,988 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:34:34,989 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: keepalive +2025-01-04 22:34:34,990 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: defer +2025-01-04 22:34:34,990 INFO SenderThread:2037873 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-04 22:34:34,990 INFO SenderThread:2037873 [sender.py:transition_state():614] send defer: 5 +2025-01-04 22:34:34,990 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:34:34,990 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:34:34,990 INFO HandlerThread:2037873 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-04 22:34:34,990 DEBUG SenderThread:2037873 [sender.py:send():379] send: history +2025-01-04 22:34:34,991 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:34:34,991 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:34:34,992 DEBUG SenderThread:2037873 [sender.py:send():379] send: summary +2025-01-04 22:34:34,995 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:34:34,995 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: defer +2025-01-04 22:34:34,995 INFO SenderThread:2037873 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-04 22:34:34,996 INFO SenderThread:2037873 [sender.py:transition_state():614] send defer: 6 +2025-01-04 22:34:34,996 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:34:34,996 INFO HandlerThread:2037873 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-04 22:34:34,996 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: defer +2025-01-04 22:34:34,996 INFO SenderThread:2037873 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-04 22:34:34,999 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:34:35,134 INFO SenderThread:2037873 [sender.py:transition_state():614] send defer: 7 +2025-01-04 22:34:35,134 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:34:35,134 INFO HandlerThread:2037873 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-04 22:34:35,134 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: defer +2025-01-04 22:34:35,135 INFO SenderThread:2037873 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-04 22:34:35,192 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/config.yaml +2025-01-04 22:34:35,192 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/wandb-summary.json +2025-01-04 22:34:35,207 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:34:37,751 INFO SenderThread:2037873 [sender.py:transition_state():614] send defer: 8 +2025-01-04 22:34:37,752 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:34:37,752 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:34:37,752 INFO HandlerThread:2037873 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-04 22:34:37,752 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: defer +2025-01-04 22:34:37,752 INFO SenderThread:2037873 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-04 22:34:37,752 INFO SenderThread:2037873 [job_builder.py:build():440] Attempting to build job artifact +2025-01-04 22:34:37,754 INFO SenderThread:2037873 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-04 22:34:37,756 INFO SenderThread:2037873 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-04 22:34:37,770 INFO SenderThread:2037873 [sender.py:transition_state():614] send defer: 9 +2025-01-04 22:34:37,770 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:34:37,770 INFO HandlerThread:2037873 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-04 22:34:37,770 DEBUG SenderThread:2037873 [sender.py:send():379] send: artifact +2025-01-04 22:34:37,982 INFO SenderThread:2037873 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxOTExMTc3MQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTE5MjI0Ng==', 'versionIndex': 420}}} +2025-01-04 22:34:37,982 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: defer +2025-01-04 22:34:37,982 INFO SenderThread:2037873 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-04 22:34:37,982 INFO SenderThread:2037873 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-04 22:34:38,193 INFO SenderThread:2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:34:38,193 INFO SenderThread:2037873 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files +2025-01-04 22:34:38,193 INFO SenderThread:2037873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/config.yaml config.yaml +2025-01-04 22:34:38,193 INFO SenderThread:2037873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/wandb-summary.json wandb-summary.json +2025-01-04 22:34:38,193 INFO SenderThread:2037873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/requirements.txt requirements.txt +2025-01-04 22:34:38,193 INFO SenderThread:2037873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/wandb-metadata.json wandb-metadata.json +2025-01-04 22:34:38,193 INFO SenderThread:2037873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/events.out.tfevents.1736047719.leonmkim-ROG-Strix-G15CS-G15CS.2037421.0 events.out.tfevents.1736047719.leonmkim-ROG-Strix-G15CS-G15CS.2037421.0 +2025-01-04 22:34:38,194 INFO SenderThread:2037873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log output.log +2025-01-04 22:34:38,194 INFO SenderThread:2037873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/diff.patch diff.patch +2025-01-04 22:34:38,198 INFO SenderThread:2037873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-04 22:34:38,199 INFO SenderThread:2037873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval/2_eval_2_fef06f9228f28f137565.mp4 media/videos/eval/2_eval_2_fef06f9228f28f137565.mp4 +2025-01-04 22:34:38,199 INFO SenderThread:2037873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval/4_eval_4_d691d8c6f8564711e47c.mp4 media/videos/eval/4_eval_4_d691d8c6f8564711e47c.mp4 +2025-01-04 22:34:38,199 INFO SenderThread:2037873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval/1_eval_1_fcc2c84034b2f80493e2.mp4 media/videos/eval/1_eval_1_fcc2c84034b2f80493e2.mp4 +2025-01-04 22:34:38,199 INFO SenderThread:2037873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval/0_eval_0_3f829840c9d5231f7c2a.mp4 media/videos/eval/0_eval_0_3f829840c9d5231f7c2a.mp4 +2025-01-04 22:34:38,199 INFO SenderThread:2037873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval/3_eval_3_2ceb841f3e55434b9bc7.mp4 media/videos/eval/3_eval_3_2ceb841f3e55434b9bc7.mp4 +2025-01-04 22:34:38,201 INFO SenderThread:2037873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/table/eval/total_success_rate_ci_6_157933031db0c923cb36.table.json media/table/eval/total_success_rate_ci_6_157933031db0c923cb36.table.json +2025-01-04 22:34:38,201 INFO SenderThread:2037873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/table/eval/success_rate_ci_5_2d447e7da5390d365153.table.json media/table/eval/success_rate_ci_5_2d447e7da5390d365153.table.json +2025-01-04 22:34:38,201 INFO SenderThread:2037873 [sender.py:transition_state():614] send defer: 10 +2025-01-04 22:34:38,202 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:34:38,202 INFO HandlerThread:2037873 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-04 22:34:38,203 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: defer +2025-01-04 22:34:38,203 INFO SenderThread:2037873 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-04 22:34:38,203 INFO SenderThread:2037873 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 22:34:38,208 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:34:38,208 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:34:38,373 INFO wandb-upload_0:2037873 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/requirements.txt +2025-01-04 22:34:38,387 INFO wandb-upload_1:2037873 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/config.yaml +2025-01-04 22:34:38,395 INFO wandb-upload_2:2037873 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/wandb-summary.json +2025-01-04 22:34:38,446 INFO wandb-upload_4:2037873 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:34:38,468 INFO wandb-upload_3:2037873 [upload_job.py:push():130] Uploaded file /tmp/tmpn9pnfdm3wandb/vb529jnn-events.out.tfevents.1736047719.leonmkim-ROG-Strix-G15CS-G15CS.2037421.0 +2025-01-04 22:34:38,669 INFO Thread-11 (_thread_body):2037873 [sender.py:transition_state():614] send defer: 11 +2025-01-04 22:34:38,669 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:34:38,669 INFO HandlerThread:2037873 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-04 22:34:38,669 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: defer +2025-01-04 22:34:38,670 INFO SenderThread:2037873 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-04 22:34:38,670 INFO SenderThread:2037873 [file_pusher.py:join():175] waiting for file pusher +2025-01-04 22:34:38,671 INFO SenderThread:2037873 [sender.py:transition_state():614] send defer: 12 +2025-01-04 22:34:38,671 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:34:38,671 INFO HandlerThread:2037873 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-04 22:34:38,671 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: defer +2025-01-04 22:34:38,671 INFO SenderThread:2037873 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-04 22:34:38,671 INFO SenderThread:2037873 [file_stream.py:finish():601] file stream finish called +2025-01-04 22:34:38,741 INFO SenderThread:2037873 [file_stream.py:finish():605] file stream finish is done +2025-01-04 22:34:38,741 INFO SenderThread:2037873 [sender.py:transition_state():614] send defer: 13 +2025-01-04 22:34:38,741 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:34:38,741 INFO HandlerThread:2037873 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-04 22:34:38,741 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: defer +2025-01-04 22:34:38,741 INFO SenderThread:2037873 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-04 22:34:38,741 INFO SenderThread:2037873 [sender.py:transition_state():614] send defer: 14 +2025-01-04 22:34:38,742 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:34:38,742 INFO HandlerThread:2037873 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-04 22:34:38,742 DEBUG SenderThread:2037873 [sender.py:send():379] send: final +2025-01-04 22:34:38,742 DEBUG SenderThread:2037873 [sender.py:send():379] send: footer +2025-01-04 22:34:38,742 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: defer +2025-01-04 22:34:38,742 INFO SenderThread:2037873 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-04 22:34:38,743 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:34:38,743 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:34:38,743 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: server_info +2025-01-04 22:34:38,743 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: server_info +2025-01-04 22:34:38,744 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:34:38,745 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: get_summary +2025-01-04 22:34:38,745 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-04 22:34:38,746 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:38,794 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:34:38,794 INFO MainThread:2037873 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-04 22:34:38,794 INFO MainThread:2037873 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-04 22:34:38,794 INFO MainThread:2037873 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-04 22:34:38,795 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: shutdown +2025-01-04 22:34:38,795 INFO HandlerThread:2037873 [handler.py:finish():882] shutting down handler +2025-01-04 22:34:39,745 INFO WriterThread:2037873 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/run-iqjkgl2e.wandb +2025-01-04 22:34:39,794 INFO SenderThread:2037873 [sender.py:finish():1615] shutting down sender +2025-01-04 22:34:39,794 INFO SenderThread:2037873 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 22:34:39,794 INFO SenderThread:2037873 [file_pusher.py:join():175] waiting for file pusher diff --git a/iqjkgl2e/wandb/debug.log b/iqjkgl2e/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..a53a864479784676dd04cd246ef5d2a38fb3315f --- /dev/null +++ b/iqjkgl2e/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-04 22:28:38,817 INFO MainThread:2037421 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-04 22:28:38,817 INFO MainThread:2037421 [wandb_setup.py:_flush():76] Configure stats pid to 2037421 +2025-01-04 22:28:38,817 INFO MainThread:2037421 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-04 22:28:38,817 INFO MainThread:2037421 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/settings +2025-01-04 22:28:38,817 INFO MainThread:2037421 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-04 22:28:38,817 INFO MainThread:2037421 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-04 22:28:38,817 INFO MainThread:2037421 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-04 22:28:38,817 INFO MainThread:2037421 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-04 22:28:38,817 INFO MainThread:2037421 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/logs/debug.log +2025-01-04 22:28:38,817 INFO MainThread:2037421 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/logs/debug-internal.log +2025-01-04 22:28:38,817 INFO MainThread:2037421 [wandb_init.py:init():569] calling init triggers +2025-01-04 22:28:38,817 INFO MainThread:2037421 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1926_1', 'wandb_notes': '1926_1_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e'} +2025-01-04 22:28:38,817 INFO MainThread:2037421 [wandb_init.py:init():619] starting backend +2025-01-04 22:28:38,818 INFO MainThread:2037421 [wandb_init.py:init():623] setting up manager +2025-01-04 22:28:38,822 INFO MainThread:2037421 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-04 22:28:38,822 INFO MainThread:2037421 [wandb_init.py:init():631] backend started and connected +2025-01-04 22:28:38,835 INFO MainThread:2037421 [wandb_init.py:init():720] updated telemetry +2025-01-04 22:28:38,843 INFO MainThread:2037421 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-04 22:28:39,090 INFO MainThread:2037421 [wandb_run.py:_on_init():2435] communicating current version +2025-01-04 22:28:39,131 INFO MainThread:2037421 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-04 22:28:39,132 INFO MainThread:2037421 [wandb_init.py:init():804] starting run threads in backend +2025-01-04 22:28:39,488 INFO MainThread:2037421 [wandb_run.py:_console_start():2413] atexit reg +2025-01-04 22:28:39,488 INFO MainThread:2037421 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-04 22:28:39,488 INFO MainThread:2037421 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-04 22:28:39,489 INFO MainThread:2037421 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-04 22:28:39,490 INFO MainThread:2037421 [wandb_init.py:init():847] run started, returning control to user process +2025-01-04 22:28:39,490 INFO MainThread:2037421 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/tb, True +2025-01-04 22:28:52,929 INFO MainThread:2037421 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2025-01-04 22:34:39,795 WARNING MsgRouterThr:2037421 [router.py:message_loop():77] message_loop has been closed diff --git a/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/code/FISH/eval_robot.py b/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/config.yaml b/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7d5151de51a6456dd2aa17d57ce8215bd2399057 --- /dev/null +++ b/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 1 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1926_1' +wandb_notes: + desc: null + value: 1926_1_restarted_5 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 1 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736047718 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: hbm +left_book_slot: + desc: null + value: twodim diff --git a/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/diff.patch b/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..b5be3e37ff49c316e2685f8bd605106c30e9bfb6 --- /dev/null +++ b/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/diff.patch @@ -0,0 +1,56 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 57a851f..27e7d93 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -138,12 +138,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' ++# wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1006_0' ++wandb_run_id: '1926_1' # seed 1 ++# wandb_run_id: '1930_0' # dataset shuffle seed 1 ++ ++ + + # RGBD+mask+contact(contact+DTC)+act history +-wandb_run_id: '1745_0' ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 71a5d1c..c7eadce 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -159,8 +159,10 @@ run_id_list = [ + ] + + run_id_list = [ +- # '1006_0', +- '1745_0' ++ # '1922_1', ++ # '1922_2', ++ '1926_1', ++ '1930_0', + ] + + checkpoint_epoch = 12000 +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index 6f97e26..285afc8 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,8 +330,8 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.005 +- self.move_to_rot_tolerance = 1.0 ++ self.move_to_trans_tolerance = 0.01 ++ self.move_to_rot_tolerance = 2.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) + self.move_to_target_timeout = 5.0 diff --git a/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/table/eval/success_rate_ci_5_2d447e7da5390d365153.table.json b/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/table/eval/success_rate_ci_5_2d447e7da5390d365153.table.json new file mode 100644 index 0000000000000000000000000000000000000000..487ca02c66feb7a1705ed4827a7a884ec91cf1f6 --- /dev/null +++ b/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/table/eval/success_rate_ci_5_2d447e7da5390d365153.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.2, 0.023880511617516982, 0.7333634495268442, 0.015644066132618713, 0.6888955831149919, 0.02994169304235303, 0.5159374773470475, 0.021239768809464744, 0.4524847120141646]]} \ No newline at end of file diff --git a/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/table/eval/total_success_rate_ci_6_157933031db0c923cb36.table.json b/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/table/eval/total_success_rate_ci_6_157933031db0c923cb36.table.json new file mode 100644 index 0000000000000000000000000000000000000000..00b55b48f68558472bb4cabb54f9b8a88d22c1ae --- /dev/null +++ b/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/table/eval/total_success_rate_ci_6_157933031db0c923cb36.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[27, 60, 0.45, 0.3149867058170319, 0.5956960320281325, 0.3247884661309667, 0.5717679709052554, 0.34116975221234236, 0.5574955567602561, 0.37607003378366466, 0.5365587621854144]]} \ No newline at end of file diff --git a/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval/0_eval_0_3f829840c9d5231f7c2a.mp4 b/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval/0_eval_0_3f829840c9d5231f7c2a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fc76a00bec747f97d8a70f8fe7894c625d11093b --- /dev/null +++ b/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval/0_eval_0_3f829840c9d5231f7c2a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f829840c9d5231f7c2a1ef7d0f2ce345f32e53845f431d6604f993f0feabd02 +size 1518861 diff --git a/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval/1_eval_1_fcc2c84034b2f80493e2.mp4 b/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval/1_eval_1_fcc2c84034b2f80493e2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..79cfadcb3576011dd9631bd542899deffde14a88 --- /dev/null +++ b/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval/1_eval_1_fcc2c84034b2f80493e2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fcc2c84034b2f80493e2c9bcfdfe89f5533ab66e5acaa9e5a19c94d4ab517e9c +size 1504602 diff --git a/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval/2_eval_2_fef06f9228f28f137565.mp4 b/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval/2_eval_2_fef06f9228f28f137565.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9f29cec42243042ccb38777f332c13626b86dc04 --- /dev/null +++ b/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval/2_eval_2_fef06f9228f28f137565.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fef06f9228f28f1375655e2e1fa944ee49dc23e690f3c86ce1bc64db99b990e5 +size 1528137 diff --git a/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval/3_eval_3_2ceb841f3e55434b9bc7.mp4 b/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval/3_eval_3_2ceb841f3e55434b9bc7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..641c294369bca984db8bc5718d9785ba63a5a006 --- /dev/null +++ b/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval/3_eval_3_2ceb841f3e55434b9bc7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ceb841f3e55434b9bc762ea3190b2ad5c6886d8ad25c7c25631e6f71967352c +size 1544200 diff --git a/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval/4_eval_4_d691d8c6f8564711e47c.mp4 b/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval/4_eval_4_d691d8c6f8564711e47c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..eb83a6c9667d9fc55685dce895dd6536e1d184c5 --- /dev/null +++ b/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval/4_eval_4_d691d8c6f8564711e47c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d691d8c6f8564711e47c11a3353dd90ba7d94653918a5c63f47690e8a8b9c0ae +size 1542124 diff --git a/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log b/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..b466eb52e37c859265dcf386989131cfac21a445 --- /dev/null +++ b/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log @@ -0,0 +1,286 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736047732.930470]: resetting environment +[INFO] [1736047732.934298]: cleared current plan +[INFO] [1736047732.935341]: moving to home +[INFO] [1736047733.937334]: reached home +[INFO] [1736047733.938552]: reset action history +[INFO] [1736047736.649518]: environment reset +Starting episode 0 +[INFO] [1736047736.653804]: resetting environment +[INFO] [1736047736.657490]: cleared current plan +[INFO] [1736047736.658463]: moving to home +[INFO] [1736047737.661025]: reached home +[INFO] [1736047737.662298]: reset action history +[INFO] [1736047740.372145]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736047766.769486]: resetting environment +[INFO] [1736047766.774007]: cleared current plan +[INFO] [1736047766.774906]: moving to home +[INFO] [1736047767.777112]: reached home +[INFO] [1736047767.778300]: reset action history +[INFO] [1736047770.488182]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/episode_rosbags/episode_0_2025-01-04-22-29-30.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2037421.log +started roslaunch server http://158.130.50.37:35261/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2038015] +process[print_text-2]: started with pid [2038039] +process[rosbag_record-3]: started with pid [2038040] +started bagging! +For topic gripper_width: timestamp difference is -102876698 for nearest: 1736047789051517690 - target: 1736047789154394388 at idx 245 +For topic gripper_width: timestamp difference is 103978243 for nearest: 1736047792618175593 - target: 1736047792514197350 at idx 249 +For topic gripper_width: timestamp difference is 108314119 for nearest: 1736047794718207918 - target: 1736047794609893799 at idx 249 +[INFO] [1736047805.309972]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736047806.055625]: Stored episode 1. +[INFO] [1736047806.055912]: Saving video... +[INFO] [1736047806.268205]: Video saved! +Starting episode 1 +[INFO] [1736047806.287283]: resetting environment +[INFO] [1736047806.298648]: cleared current plan +[INFO] [1736047806.299464]: moving to home +[INFO] [1736047809.501262]: reached home +[INFO] [1736047809.501757]: reset action history +[INFO] [1736047812.213083]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736047829.524517]: resetting environment +[INFO] [1736047829.527769]: cleared current plan +[INFO] [1736047829.527943]: moving to home +[INFO] [1736047830.529251]: reached home +[INFO] [1736047830.530109]: reset action history +[INFO] [1736047833.243020]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/episode_rosbags/episode_1_2025-01-04-22-30-33.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2037421.log +started roslaunch server http://158.130.50.37:35825/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2038178] +process[print_text-5]: started with pid [2038179] +process[rosbag_record-6]: started with pid [2038180] +started bagging! +For topic gripper_width: timestamp difference is 110065697 for nearest: 1736047842118180750 - target: 1736047842008115053 at idx 120 +For topic gripper_width: timestamp difference is 111105840 for nearest: 1736047843418176334 - target: 1736047843307070494 at idx 137 +For topic gripper_width: timestamp difference is 115735262 for nearest: 1736047849118203848 - target: 1736047849002468586 at idx 214 +For topic gripper_width: timestamp difference is -114030424 for nearest: 1736047853351533589 - target: 1736047853465564013 at idx 249 +[INFO] [1736047868.139350]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1736047868.898199]: Stored episode 2. +[INFO] [1736047868.912596]: Saving video... +[WARN] [1736047869.518129]: Plan exhausted +[INFO] [1736047869.494633]: Video saved! +Starting episode 2 +[INFO] [1736047869.536663]: resetting environment +[INFO] [1736047869.550113]: cleared current plan +[INFO] [1736047869.550409]: moving to home +[INFO] [1736047872.753206]: reached home +[INFO] [1736047872.757155]: reset action history +[INFO] [1736047875.535188]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736047886.221068]: resetting environment +[INFO] [1736047886.233594]: cleared current plan +[INFO] [1736047886.234125]: moving to home +[INFO] [1736047887.239157]: reached home +[INFO] [1736047887.239652]: reset action history +[INFO] [1736047889.947892]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/episode_rosbags/episode_2_2025-01-04-22-31-29.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2037421.log +started roslaunch server http://158.130.50.37:38983/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2038323] +process[print_text-8]: started with pid [2038324] +process[rosbag_record-9]: started with pid [2038348] +started bagging! +For topic gripper_width: timestamp difference is 105597820 for nearest: 1736047919051534977 - target: 1736047918945937157 at idx 249 +For topic gripper_width: timestamp difference is -126705661 for nearest: 1736047920151507602 - target: 1736047920278213263 at idx 249 +[INFO] [1736047924.974254]: Storing episode... +[print_text-8] killing on exit[rosbag_record-9] killing on exit +[pub_text-7] killing on exit +[WARN] [1736047926.316970]: Plan exhausted +[WARN] [1736047926.357560]: Plan exhausted +[INFO] [1736047925.700742]: Stored episode 3. +[INFO] [1736047925.705294]: Saving video... +[INFO] [1736047926.301287]: Video saved! +Starting episode 3 +[INFO] [1736047926.342418]: resetting environment +[INFO] [1736047926.370667]: cleared current plan +[INFO] [1736047926.370921]: moving to home +[INFO] [1736047929.773833]: reached home +[INFO] [1736047929.777109]: reset action history +[INFO] [1736047932.508707]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736047948.076141]: resetting environment +[INFO] [1736047948.093113]: cleared current plan +[INFO] [1736047948.103714]: moving to home +[INFO] [1736047949.108570]: reached home +[INFO] [1736047949.110809]: reset action history +[INFO] [1736047951.842161]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/episode_rosbags/episode_3_2025-01-04-22-32-31.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2037421.log +started roslaunch server http://158.130.50.37:34657/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2038490] +process[print_text-11]: started with pid [2038514] +process[rosbag_record-12]: started with pid [2038515] +started bagging! +For topic gripper_width: timestamp difference is 101110297 for nearest: 1736047954718178111 - target: 1736047954617067814 at idx 44 +For topic gripper_width: timestamp difference is 102127279 for nearest: 1736047956018174375 - target: 1736047955916047096 at idx 62 +For topic gripper_width: timestamp difference is 103543618 for nearest: 1736047957851507523 - target: 1736047957747963905 at idx 87 +For topic gripper_width: timestamp difference is 108348771 for nearest: 1736047963818178101 - target: 1736047963709829330 at idx 163 +[INFO] [1736047986.846260]: Storing episode... +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[print_text-11] killing on exit +[WARN] [1736047988.161441]: Plan exhausted +[WARN] [1736047988.201450]: Plan exhausted +[WARN] [1736047988.237343]: Plan exhausted +[WARN] [1736047988.278009]: Plan exhausted +[WARN] [1736047988.317370]: Plan exhausted +[WARN] [1736047988.360298]: Plan exhausted +[WARN] [1736047988.397373]: Plan exhausted +[INFO] [1736047987.715964]: Stored episode 4. +[INFO] [1736047987.722782]: Saving video... +[INFO] [1736047988.333117]: Video saved! +Starting episode 4 +[INFO] [1736047988.384646]: resetting environment +[INFO] [1736047988.406676]: cleared current plan +[INFO] [1736047988.407145]: moving to home +[INFO] [1736047992.009765]: reached home +[INFO] [1736047992.010873]: reset action history +[INFO] [1736047994.797018]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736047997.996882]: resetting environment +[INFO] [1736047998.017136]: cleared current plan +[INFO] [1736047998.017416]: moving to home +[INFO] [1736047999.019633]: reached home +[INFO] [1736047999.024350]: reset action history +[INFO] [1736048001.754799]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/episode_rosbags/episode_4_2025-01-04-22-33-21.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2037421.log +started roslaunch server http://158.130.50.37:44055/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2038658] +process[print_text-14]: started with pid [2038659] +process[rosbag_record-15]: started with pid [2038660] +started bagging! +For topic gripper_width: timestamp difference is 108244510 for nearest: 1736048004818211408 - target: 1736048004709966898 at idx 42 +For topic gripper_width: timestamp difference is 103683364 for nearest: 1736048036318174970 - target: 1736048036214491606 at idx 249 +[INFO] [1736048036.843203]: Storing episode... +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit[print_text-14] killing on exit +[WARN] [1736048038.164550]: Plan exhausted +[WARN] [1736048038.197299]: Plan exhausted +[INFO] [1736048037.668376]: Stored episode 5. +[INFO] [1736048037.672350]: Saving video... +[INFO] [1736048038.107420]: Video saved! +[INFO] [1736048038.173436]: resetting environment +[INFO] [1736048038.206722]: cleared current plan +[INFO] [1736048038.207478]: moving to home +[INFO] [1736048041.717390]: reached home +[INFO] [1736048041.726399]: reset action history +[INFO] [1736048044.515815]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/requirements.txt b/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/wandb-metadata.json b/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..c07c7667c36f36f14ee881b8c0b64c0401be8cda --- /dev/null +++ b/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-05T03:28:39.155554", + "startedAt": "2025-01-05T03:28:38.811012", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "02cf07d901f3d8187a302cd51c9faa80321417ee" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3560.90725, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 4495.482, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4493.257, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 762.5235061645508 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/wandb-summary.json b/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..d5d256897dc7c1d9e1ad9f5990051852a6b5eeea --- /dev/null +++ b/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "3f829840c9d5231f7c2a1ef7d0f2ce345f32e53845f431d6604f993f0feabd02", "size": 1518861, "path": "media/videos/eval/0_eval_0_3f829840c9d5231f7c2a.mp4"}, "global_step": 665, "_timestamp": 1736048051.473574, "_runtime": 332.6505479812622, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "fcc2c84034b2f80493e2c9bcfdfe89f5533ab66e5acaa9e5a19c94d4ab517e9c", "size": 1504602, "path": "media/videos/eval/1_eval_1_fcc2c84034b2f80493e2.mp4"}, "eval/num_success": 1.0, "episode": 60.0, "eval/success_rate": 0.20000000298023224, "eval/success_rate_lb_0.01": 0.02388051152229309, "eval/success_rate_ub_0.01": 0.7333634495735168, "eval/success_rate_lb_0.025": 0.015644066035747528, "eval/success_rate_ub_0.025": 0.688895583152771, "eval/success_rate_lb_0.05": 0.02994169294834137, "eval/success_rate_ub_0.05": 0.5159374475479126, "eval/success_rate_lb_0.1": 0.021239768713712692, "eval/success_rate_ub_0.1": 0.45248472690582275, "eval/2_eval": {"_type": "video-file", "sha256": "fef06f9228f28f1375655e2e1fa944ee49dc23e690f3c86ce1bc64db99b990e5", "size": 1528137, "path": "media/videos/eval/2_eval_2_fef06f9228f28f137565.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "2ceb841f3e55434b9bc762ea3190b2ad5c6886d8ad25c7c25631e6f71967352c", "size": 1544200, "path": "media/videos/eval/3_eval_3_2ceb841f3e55434b9bc7.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "d691d8c6f8564711e47c11a3353dd90ba7d94653918a5c63f47690e8a8b9c0ae", "size": 1542124, "path": "media/videos/eval/4_eval_4_d691d8c6f8564711e47c.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "2d447e7da5390d365153b465cab4c070d4dcc878bdbcba3e92cb6f4e90800649", "size": 405, "artifact_path": "wandb-client-artifact://2nc7r0tttuapa4a1aztstjpzm210e1nirvc0putahkamtt242a31p46a6l0der8qofeeglzj4jko4c6k5w36rove72vdacmux86zksqp7pcfe8qs2uvwb2prwegvlydh/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://v0xgls0r6bkofal1iomx5k4t43zf1nmkwyvfj100z0dcudylw7fu93v25qdrj86jgcwm2c9iaewy3mbohky1derp5b1bxh8skj1x5f23gm53ai4h4azud33tnfvroja6:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_2d447e7da5390d365153.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 27, "total_num_episodes": 60, "total_success_rate": 0.45, "total_success_rate_lb_0.01": 0.3149867058170319, "total_success_rate_ub_0.01": 0.5956960320281325, "total_success_rate_lb_0.025": 0.3247884661309667, "total_success_rate_ub_0.025": 0.5717679709052554, "total_success_rate_lb_0.05": 0.34116975221234236, "total_success_rate_ub_0.05": 0.5574955567602561, "total_success_rate_lb_0.1": 0.37607003378366466, "total_success_rate_ub_0.1": 0.5365587621854144, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "157933031db0c923cb3647c6d355aaaef888ac308402f36c306d7bc24b254992", "size": 508, "artifact_path": "wandb-client-artifact://gvtwoj57fwlbp2qetn1kc2yb5rxq0ly02io20czijl3bjs8575g6k6d7u7j5kn24t6b2tlkeixpsfrdc5u2a9mpzvmdwolmqnn4qorb3japkzcv6xjktmoyg4l8h71ma/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://2ssxw2kcj70a9bkti73ej912owl5luszb9yo2xxdhm7pjocqq6hkitypjjoo98dytjp5nlld6cbksbtfq1vm2hkqm4ygn4g2kdqns2773zka8sfhfttpoj7rbl1eaczd:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_157933031db0c923cb36.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 349}, "eval/total_success_rate": 0.44999998807907104, "eval/total_success_rate_lb_0.01": 0.3149867057800293, "eval/total_success_rate_ub_0.01": 0.5956960320472717, "eval/total_success_rate_lb_0.025": 0.32478848099708557, "eval/total_success_rate_ub_0.025": 0.5717679858207703, "eval/total_success_rate_lb_0.05": 0.3411697447299957, "eval/total_success_rate_ub_0.05": 0.5574955344200134, "eval/total_success_rate_lb_0.1": 0.3760700225830078, "eval/total_success_rate_ub_0.1": 0.5365587472915649} \ No newline at end of file diff --git a/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/logs/debug-internal.log b/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..999ec1f7138dc7232f27cce471baf7750f87bad9 --- /dev/null +++ b/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/logs/debug-internal.log @@ -0,0 +1,886 @@ +2025-01-04 22:28:38,823 INFO StreamThr :2037873 [internal.py:wandb_internal():85] W&B internal server running at pid: 2037873, started at: 2025-01-04 22:28:38.822989 +2025-01-04 22:28:38,824 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status +2025-01-04 22:28:38,835 INFO WriterThread:2037873 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/run-iqjkgl2e.wandb +2025-01-04 22:28:38,835 DEBUG SenderThread:2037873 [sender.py:send():379] send: header +2025-01-04 22:28:38,845 DEBUG SenderThread:2037873 [sender.py:send():379] send: run +2025-01-04 22:28:39,074 INFO SenderThread:2037873 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files +2025-01-04 22:28:39,074 INFO SenderThread:2037873 [sender.py:_start_run_threads():1188] run started: iqjkgl2e with start time 1736047718.823026 +2025-01-04 22:28:39,090 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: check_version +2025-01-04 22:28:39,090 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: check_version +2025-01-04 22:28:39,138 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: run_start +2025-01-04 22:28:39,143 DEBUG HandlerThread:2037873 [system_info.py:__init__():26] System info init +2025-01-04 22:28:39,143 DEBUG HandlerThread:2037873 [system_info.py:__init__():41] System info init done +2025-01-04 22:28:39,143 INFO HandlerThread:2037873 [system_monitor.py:start():194] Starting system monitor +2025-01-04 22:28:39,143 INFO SystemMonitor:2037873 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-04 22:28:39,143 INFO HandlerThread:2037873 [system_monitor.py:probe():214] Collecting system info +2025-01-04 22:28:39,144 INFO SystemMonitor:2037873 [interfaces.py:start():188] Started cpu monitoring +2025-01-04 22:28:39,144 INFO SystemMonitor:2037873 [interfaces.py:start():188] Started disk monitoring +2025-01-04 22:28:39,145 INFO SystemMonitor:2037873 [interfaces.py:start():188] Started gpu monitoring +2025-01-04 22:28:39,145 INFO SystemMonitor:2037873 [interfaces.py:start():188] Started memory monitoring +2025-01-04 22:28:39,145 INFO SystemMonitor:2037873 [interfaces.py:start():188] Started network monitoring +2025-01-04 22:28:39,155 DEBUG HandlerThread:2037873 [system_info.py:probe():152] Probing system +2025-01-04 22:28:39,156 DEBUG HandlerThread:2037873 [system_info.py:_probe_git():137] Probing git +2025-01-04 22:28:39,161 DEBUG HandlerThread:2037873 [system_info.py:_probe_git():145] Probing git done +2025-01-04 22:28:39,161 DEBUG HandlerThread:2037873 [system_info.py:probe():200] Probing system done +2025-01-04 22:28:39,161 DEBUG HandlerThread:2037873 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-05T03:28:39.155554', 'startedAt': '2025-01-05T03:28:38.811012', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '02cf07d901f3d8187a302cd51c9faa80321417ee'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3560.90725, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4495.482, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4493.257, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 762.5235061645508}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-04 22:28:39,161 INFO HandlerThread:2037873 [system_monitor.py:probe():224] Finished collecting system info +2025-01-04 22:28:39,161 INFO HandlerThread:2037873 [system_monitor.py:probe():227] Publishing system info +2025-01-04 22:28:39,161 DEBUG HandlerThread:2037873 [system_info.py:_save_code():44] Saving code +2025-01-04 22:28:39,168 DEBUG HandlerThread:2037873 [system_info.py:_save_code():67] Saving code done +2025-01-04 22:28:39,168 DEBUG HandlerThread:2037873 [system_info.py:_save_patches():84] Saving git patches +2025-01-04 22:28:39,233 DEBUG HandlerThread:2037873 [system_info.py:_save_patches():126] Saving git patches done +2025-01-04 22:28:39,233 INFO HandlerThread:2037873 [system_monitor.py:probe():229] Finished publishing system info +2025-01-04 22:28:39,237 DEBUG SenderThread:2037873 [sender.py:send():379] send: files +2025-01-04 22:28:39,237 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-04 22:28:39,237 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-04 22:28:39,238 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-04 22:28:39,488 INFO wandb-upload_0:2037873 [upload_job.py:push():130] Uploaded file /tmp/tmpn9pnfdm3wandb/poa4dgwj-wandb-metadata.json +2025-01-04 22:28:39,488 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: python_packages +2025-01-04 22:28:39,488 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: python_packages +2025-01-04 22:28:39,488 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:39,489 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:28:39,489 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:28:39,504 INFO wandb-upload_2:2037873 [upload_job.py:push():130] Uploaded file /tmp/tmpn9pnfdm3wandb/mm48v6aw-diff.patch +2025-01-04 22:28:39,533 INFO HandlerThread:2037873 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/tb" + save: true +} +control { + relay_id: "b649c796fe424680bb0ddd3c2c946087" +} +_info { + stream_id: "iqjkgl2e" +} + +2025-01-04 22:28:39,533 DEBUG HandlerThread:2037873 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-04 22:28:39,537 INFO wandb-upload_1:2037873 [upload_job.py:push():130] Uploaded file /tmp/tmpn9pnfdm3wandb/b6rj74pm-code/FISH/eval_robot.py +2025-01-04 22:28:39,617 DEBUG SenderThread:2037873 [sender.py:send():379] send: telemetry +2025-01-04 22:28:39,617 DEBUG SenderThread:2037873 [sender.py:send():379] send: tbrecord +2025-01-04 22:28:39,618 DEBUG SenderThread:2037873 [sender.py:send():379] send: files +2025-01-04 22:28:39,618 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file events.out.tfevents.1736047719.leonmkim-ROG-Strix-G15CS-G15CS.2037421.0 with policy live +2025-01-04 22:28:39,804 INFO wandb-upload_0:2037873 [upload_job.py:push():130] Uploaded file /tmp/tmpn9pnfdm3wandb/ryljuk5s-events.out.tfevents.1736047719.leonmkim-ROG-Strix-G15CS-G15CS.2037421.0 +2025-01-04 22:28:40,076 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:28:40,076 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/wandb-metadata.json +2025-01-04 22:28:40,076 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/diff.patch +2025-01-04 22:28:40,076 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/requirements.txt +2025-01-04 22:28:40,077 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/events.out.tfevents.1736047719.leonmkim-ROG-Strix-G15CS-G15CS.2037421.0 +2025-01-04 22:28:40,077 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/code/FISH/eval_robot.py +2025-01-04 22:28:40,077 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/code +2025-01-04 22:28:40,077 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/code/FISH +2025-01-04 22:28:40,488 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:41,488 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:42,075 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:28:42,488 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:43,488 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:44,488 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:28:44,489 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:45,489 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:46,489 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:47,489 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:48,489 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:49,488 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:28:49,489 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:50,489 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:51,489 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:52,490 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:52,930 DEBUG SenderThread:2037873 [sender.py:send():379] send: config +2025-01-04 22:28:53,490 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:54,078 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:28:54,488 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:28:54,488 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:28:54,490 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:54,620 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:28:55,490 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:56,079 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:28:56,490 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:57,490 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:58,080 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:28:58,490 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:28:59,490 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:00,081 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:29:00,372 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:29:00,490 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:01,490 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:02,082 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:29:02,491 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:03,491 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:04,491 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:05,489 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:29:05,491 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:06,491 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:07,491 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:08,491 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:09,488 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:29:09,488 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:29:09,529 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:10,491 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:10,647 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:29:11,084 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/config.yaml +2025-01-04 22:29:11,491 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:12,492 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:13,492 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:14,492 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:15,493 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:16,489 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:29:16,493 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:17,493 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:18,493 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:19,493 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:20,493 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:21,490 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:29:21,493 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:22,493 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:23,494 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:24,488 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:29:24,488 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:29:24,533 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:25,494 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:26,494 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:26,596 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:29:27,494 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:28,089 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:29:28,494 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:29,494 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:30,090 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:29:30,529 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:31,495 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:31,779 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:29:32,090 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:29:32,495 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:33,495 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:34,495 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:35,495 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:36,495 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:37,490 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:29:37,495 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:38,495 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:39,146 DEBUG SystemMonitor:2037873 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-04 22:29:39,147 DEBUG SenderThread:2037873 [sender.py:send():379] send: stats +2025-01-04 22:29:39,488 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:29:39,489 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:29:39,533 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:40,495 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:41,496 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:42,496 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:42,580 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:29:43,496 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:44,496 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:45,496 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:46,497 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:47,497 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:48,490 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:29:48,497 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:49,497 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:50,497 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:51,500 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:52,096 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:29:52,498 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:53,498 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:53,685 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:29:54,096 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:29:54,488 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:29:54,489 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:29:54,529 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:55,498 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:56,097 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:29:56,498 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:57,498 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:58,499 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:29:58,758 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:29:59,499 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:00,499 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:01,499 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:02,499 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:03,500 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:04,490 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:30:04,499 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:05,499 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:06,286 DEBUG SenderThread:2037873 [sender.py:send():379] send: files +2025-01-04 22:30:06,286 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_3f829840c9d5231f7c2a.mp4 with policy now +2025-01-04 22:30:06,286 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:30:06,287 DEBUG SenderThread:2037873 [sender.py:send():379] send: history +2025-01-04 22:30:06,287 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:30:06,288 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:30:06,501 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:06,537 INFO wandb-upload_1:2037873 [upload_job.py:push():130] Uploaded file /tmp/tmpn9pnfdm3wandb/2gejalib-media/videos/eval/0_eval_0_3f829840c9d5231f7c2a.mp4 +2025-01-04 22:30:07,100 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval/0_eval_0_3f829840c9d5231f7c2a.mp4 +2025-01-04 22:30:07,100 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/wandb-summary.json +2025-01-04 22:30:07,100 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval +2025-01-04 22:30:07,100 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media +2025-01-04 22:30:07,100 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos +2025-01-04 22:30:07,388 INFO wandb-upload_0:2037873 [upload_job.py:push():130] Uploaded file /tmp/tmpn9pnfdm3wandb/ewfh9atc-media/videos/eval/0_eval_0_3f829840c9d5231f7c2a.mp4 +2025-01-04 22:30:07,500 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:08,100 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:30:08,500 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:09,147 DEBUG SenderThread:2037873 [sender.py:send():379] send: stats +2025-01-04 22:30:09,489 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:30:09,489 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:30:09,529 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:09,608 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:30:10,500 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:11,500 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:12,101 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:30:12,500 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:13,500 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:14,102 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:30:14,501 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:15,491 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:30:15,501 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:16,501 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:17,501 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:18,501 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:19,501 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:20,491 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:30:20,501 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:21,501 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:22,501 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:23,502 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:24,489 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:30:24,489 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:30:24,529 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:25,502 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:25,619 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:30:26,502 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:27,502 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:28,503 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:29,503 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:30,107 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/events.out.tfevents.1736047719.leonmkim-ROG-Strix-G15CS-G15CS.2037421.0 +2025-01-04 22:30:30,347 INFO wandb-upload_1:2037873 [upload_job.py:push():130] Uploaded file /tmp/tmpn9pnfdm3wandb/mqkj06tu-events.out.tfevents.1736047719.leonmkim-ROG-Strix-G15CS-G15CS.2037421.0 +2025-01-04 22:30:30,503 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:31,503 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:31,577 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:30:32,108 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:30:32,503 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:33,503 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:34,504 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:35,504 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:36,110 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:30:36,504 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:37,491 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:30:37,504 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:38,504 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:39,154 DEBUG SenderThread:2037873 [sender.py:send():379] send: stats +2025-01-04 22:30:39,489 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:30:39,489 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:30:39,529 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:40,505 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:41,505 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:42,505 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:43,169 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:30:43,505 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:44,112 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:30:44,505 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:45,505 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:46,113 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:30:46,505 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:47,505 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:48,492 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:30:48,505 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:49,505 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:50,114 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:30:50,506 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:51,506 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:52,506 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:53,492 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:30:53,506 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:54,489 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:30:54,489 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:30:54,506 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:55,506 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:56,116 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:30:56,506 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:57,506 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:58,507 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:30:58,609 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:30:59,506 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:00,507 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:01,508 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:02,508 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:03,508 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:04,492 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:31:04,508 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:05,508 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:06,508 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:07,508 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:08,509 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:09,148 DEBUG SenderThread:2037873 [sender.py:send():379] send: stats +2025-01-04 22:31:09,495 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:31:09,498 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:31:09,498 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:31:09,518 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:09,561 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:31:09,562 DEBUG SenderThread:2037873 [sender.py:send():379] send: files +2025-01-04 22:31:09,562 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_fcc2c84034b2f80493e2.mp4 with policy now +2025-01-04 22:31:09,563 DEBUG SenderThread:2037873 [sender.py:send():379] send: history +2025-01-04 22:31:09,563 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:31:09,568 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:31:09,814 INFO wandb-upload_0:2037873 [upload_job.py:push():130] Uploaded file /tmp/tmpn9pnfdm3wandb/0o5eybqx-media/videos/eval/1_eval_1_fcc2c84034b2f80493e2.mp4 +2025-01-04 22:31:10,120 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:31:10,120 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/wandb-summary.json +2025-01-04 22:31:10,121 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval/1_eval_1_fcc2c84034b2f80493e2.mp4 +2025-01-04 22:31:10,121 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval +2025-01-04 22:31:10,381 INFO wandb-upload_1:2037873 [upload_job.py:push():130] Uploaded file /tmp/tmpn9pnfdm3wandb/b18t9tm8-media/videos/eval/1_eval_1_fcc2c84034b2f80493e2.mp4 +2025-01-04 22:31:10,510 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:11,510 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:12,121 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:31:12,510 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:13,510 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:14,122 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:31:14,499 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:31:14,514 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:15,512 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:16,510 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:17,511 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:18,123 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:31:18,511 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:19,511 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:20,495 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:31:20,512 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:21,511 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:22,512 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:23,512 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:24,491 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:31:24,491 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:31:24,537 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:25,513 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:25,588 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:31:26,513 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:27,000 DEBUG SenderThread:2037873 [sender.py:send():379] send: history +2025-01-04 22:31:27,000 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:31:27,001 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:31:27,126 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/wandb-summary.json +2025-01-04 22:31:27,127 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/events.out.tfevents.1736047719.leonmkim-ROG-Strix-G15CS-G15CS.2037421.0 +2025-01-04 22:31:27,512 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:28,126 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:31:28,512 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:29,513 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:30,513 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:31,193 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:31:31,513 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:32,128 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:31:32,513 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:33,513 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:34,513 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:35,513 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:36,496 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:31:36,513 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:37,513 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:38,513 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:39,149 DEBUG SenderThread:2037873 [sender.py:send():379] send: stats +2025-01-04 22:31:39,491 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:31:39,491 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:31:39,533 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:40,514 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:41,514 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:41,601 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:31:42,514 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:43,514 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:44,514 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:45,516 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:46,515 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:47,496 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:31:47,515 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:48,516 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:49,516 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:50,516 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:51,516 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:52,496 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:31:52,516 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:53,516 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:54,491 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:31:54,491 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:31:54,533 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:55,516 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:56,516 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:57,516 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:57,565 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:31:58,517 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:31:59,517 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:00,137 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:32:00,518 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:01,518 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:02,138 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:32:02,518 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:03,429 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:32:03,518 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:04,518 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:05,522 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:06,139 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:32:06,336 DEBUG SenderThread:2037873 [sender.py:send():379] send: files +2025-01-04 22:32:06,336 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_fef06f9228f28f137565.mp4 with policy now +2025-01-04 22:32:06,339 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:32:06,339 DEBUG SenderThread:2037873 [sender.py:send():379] send: history +2025-01-04 22:32:06,339 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:32:06,347 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:32:06,519 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:06,598 INFO wandb-upload_0:2037873 [upload_job.py:push():130] Uploaded file /tmp/tmpn9pnfdm3wandb/lkqj9xbf-media/videos/eval/2_eval_2_fef06f9228f28f137565.mp4 +2025-01-04 22:32:07,139 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/wandb-summary.json +2025-01-04 22:32:07,139 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval/2_eval_2_fef06f9228f28f137565.mp4 +2025-01-04 22:32:07,140 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval +2025-01-04 22:32:07,449 INFO wandb-upload_1:2037873 [upload_job.py:push():130] Uploaded file /tmp/tmpn9pnfdm3wandb/l3mqs4sy-media/videos/eval/2_eval_2_fef06f9228f28f137565.mp4 +2025-01-04 22:32:07,522 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:08,140 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:32:08,524 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:09,149 DEBUG SenderThread:2037873 [sender.py:send():379] send: stats +2025-01-04 22:32:09,149 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:32:09,493 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:32:09,493 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:32:09,537 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:10,525 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:11,532 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:12,141 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:32:12,549 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:13,523 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:14,142 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:32:14,497 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:32:14,525 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:15,525 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:16,527 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:17,533 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:18,527 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:19,498 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:32:19,537 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:20,527 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:21,528 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:22,532 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:23,534 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:24,496 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:32:24,496 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:32:24,537 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:24,581 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:32:25,530 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:26,533 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:27,535 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:28,146 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/events.out.tfevents.1736047719.leonmkim-ROG-Strix-G15CS-G15CS.2037421.0 +2025-01-04 22:32:28,385 INFO wandb-upload_0:2037873 [upload_job.py:push():130] Uploaded file /tmp/tmpn9pnfdm3wandb/cfupijsq-events.out.tfevents.1736047719.leonmkim-ROG-Strix-G15CS-G15CS.2037421.0 +2025-01-04 22:32:28,538 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:28,858 DEBUG SenderThread:2037873 [sender.py:send():379] send: history +2025-01-04 22:32:28,858 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:32:28,858 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:32:29,146 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/wandb-summary.json +2025-01-04 22:32:29,539 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:30,147 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:32:30,499 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:32:30,538 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:31,539 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:32,537 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:33,537 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:34,148 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:32:34,537 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:35,537 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:35,773 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:32:36,149 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:32:36,537 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:37,537 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:38,150 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:32:38,537 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:39,149 DEBUG SenderThread:2037873 [sender.py:send():379] send: stats +2025-01-04 22:32:39,493 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:32:39,493 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:32:39,538 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:40,150 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:32:40,538 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:41,538 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:41,621 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:32:42,538 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:43,538 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:44,538 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:45,538 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:46,153 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:32:46,539 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:47,500 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:32:47,539 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:48,539 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:49,539 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:50,539 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:51,539 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:52,500 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:32:52,540 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:53,540 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:54,493 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:32:54,494 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:32:54,540 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:55,540 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:56,540 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:57,540 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:57,566 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:32:58,540 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:32:59,540 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:00,540 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:01,541 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:02,541 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:03,500 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:33:03,543 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:04,541 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:05,541 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:06,542 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:07,549 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:08,160 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:33:08,374 DEBUG SenderThread:2037873 [sender.py:send():379] send: files +2025-01-04 22:33:08,374 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_2ceb841f3e55434b9bc7.mp4 with policy now +2025-01-04 22:33:08,378 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:33:08,379 DEBUG SenderThread:2037873 [sender.py:send():379] send: history +2025-01-04 22:33:08,379 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:33:08,381 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:33:08,549 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:08,621 INFO wandb-upload_1:2037873 [upload_job.py:push():130] Uploaded file /tmp/tmpn9pnfdm3wandb/lhgnl3iz-media/videos/eval/3_eval_3_2ceb841f3e55434b9bc7.mp4 +2025-01-04 22:33:09,150 DEBUG SenderThread:2037873 [sender.py:send():379] send: stats +2025-01-04 22:33:09,150 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:33:09,160 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/wandb-summary.json +2025-01-04 22:33:09,160 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval/3_eval_3_2ceb841f3e55434b9bc7.mp4 +2025-01-04 22:33:09,160 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval +2025-01-04 22:33:09,445 INFO wandb-upload_0:2037873 [upload_job.py:push():130] Uploaded file /tmp/tmpn9pnfdm3wandb/rezj31aj-media/videos/eval/3_eval_3_2ceb841f3e55434b9bc7.mp4 +2025-01-04 22:33:09,494 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:33:09,494 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:33:09,554 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:10,160 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:33:10,563 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:11,554 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:12,555 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:13,557 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:14,162 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:33:14,504 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:33:14,562 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:15,560 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:16,162 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:33:16,563 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:17,558 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:18,163 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/events.out.tfevents.1736047719.leonmkim-ROG-Strix-G15CS-G15CS.2037421.0 +2025-01-04 22:33:18,562 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:18,919 DEBUG SenderThread:2037873 [sender.py:send():379] send: history +2025-01-04 22:33:18,919 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:33:18,920 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:33:19,164 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/wandb-summary.json +2025-01-04 22:33:19,563 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:20,164 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:33:20,502 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:33:20,567 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:21,563 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:22,562 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:23,562 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:24,165 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:33:24,494 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:33:24,494 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:33:24,562 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:25,563 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:25,865 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:33:26,166 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:33:26,563 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:27,563 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:28,563 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:29,563 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:30,563 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:31,502 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:33:31,564 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:32,564 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:33,564 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:34,564 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:35,564 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:36,503 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:33:36,565 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:37,565 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:38,565 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:39,156 DEBUG SenderThread:2037873 [sender.py:send():379] send: stats +2025-01-04 22:33:39,494 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:33:39,494 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:33:39,565 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:40,565 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:41,565 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:33:41,565 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:42,565 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:43,565 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:44,566 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:45,566 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:46,566 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:47,502 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:33:47,566 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:48,566 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:49,566 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:50,566 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:51,566 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:52,503 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:33:52,566 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:53,566 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:54,494 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:33:54,494 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:33:54,567 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:55,567 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:56,567 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:57,574 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:57,670 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:33:58,172 DEBUG SenderThread:2037873 [sender.py:send():379] send: files +2025-01-04 22:33:58,172 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_d691d8c6f8564711e47c.mp4 with policy now +2025-01-04 22:33:58,173 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:33:58,173 DEBUG SenderThread:2037873 [sender.py:send():379] send: history +2025-01-04 22:33:58,173 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:33:58,177 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:33:58,177 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/wandb-summary.json +2025-01-04 22:33:58,177 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval/4_eval_4_d691d8c6f8564711e47c.mp4 +2025-01-04 22:33:58,177 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval +2025-01-04 22:33:58,187 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:33:58,430 INFO wandb-upload_1:2037873 [upload_job.py:push():130] Uploaded file /tmp/tmpn9pnfdm3wandb/k0tg9pjp-media/videos/eval/4_eval_4_d691d8c6f8564711e47c.mp4 +2025-01-04 22:33:58,568 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:33:58,724 INFO wandb-upload_0:2037873 [upload_job.py:push():130] Uploaded file /tmp/tmpn9pnfdm3wandb/0l4m7nbc-media/videos/eval/4_eval_4_d691d8c6f8564711e47c.mp4 +2025-01-04 22:33:59,587 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:00,178 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:34:00,574 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:01,581 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:02,579 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:02,769 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:34:03,584 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:04,179 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:34:04,588 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:05,590 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:06,180 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:34:06,590 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:07,587 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:08,557 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:34:08,598 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:09,151 DEBUG SenderThread:2037873 [sender.py:send():379] send: stats +2025-01-04 22:34:09,497 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:34:09,497 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:34:09,593 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:10,595 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:10,946 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 22:34:10,946 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 22:34:11,141 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:34:11,182 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/events.out.tfevents.1736047719.leonmkim-ROG-Strix-G15CS-G15CS.2037421.0 +2025-01-04 22:34:11,182 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/table/eval/success_rate_ci_5_2d447e7da5390d365153.table.json +2025-01-04 22:34:11,182 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media +2025-01-04 22:34:11,182 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/table/eval +2025-01-04 22:34:11,182 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/table +2025-01-04 22:34:11,394 INFO wandb-upload_0:2037873 [upload_job.py:push():130] Uploaded file /tmp/tmpn9pnfdm3wandb/qnz3jgu8-media/table/eval/success_rate_ci_5_2d447e7da5390d365153.table.json +2025-01-04 22:34:11,403 INFO wandb-upload_1:2037873 [upload_job.py:push():130] Uploaded file /tmp/tmpn9pnfdm3wandb/oe9jyqc9-events.out.tfevents.1736047719.leonmkim-ROG-Strix-G15CS-G15CS.2037421.0 +2025-01-04 22:34:11,434 INFO wandb-upload_2:2037873 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmprs_byj3h +2025-01-04 22:34:11,574 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 22:34:11,584 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 22:34:11,592 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:11,830 INFO SenderThread:2037873 [sender.py:send_request_log_artifact():1525] logged artifact run-iqjkgl2e-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE5NTkzNw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg2MzM5Mg==', 'latestArtifact': None}} +2025-01-04 22:34:11,830 DEBUG SenderThread:2037873 [sender.py:send():379] send: history +2025-01-04 22:34:11,830 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:34:11,831 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:34:11,831 DEBUG SenderThread:2037873 [sender.py:send():379] send: files +2025-01-04 22:34:11,831 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_2d447e7da5390d365153.table.json with policy now +2025-01-04 22:34:11,831 DEBUG SenderThread:2037873 [sender.py:send():379] send: history +2025-01-04 22:34:11,831 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:34:11,831 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:34:11,832 DEBUG SenderThread:2037873 [sender.py:send():379] send: telemetry +2025-01-04 22:34:11,832 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:34:11,832 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:34:11,832 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:34:11,833 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:34:11,833 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:34:11,833 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:34:11,833 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:34:11,834 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:34:11,834 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:34:11,834 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:34:11,834 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:34:11,835 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:34:11,835 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:34:11,837 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:34:11,837 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:34:11,837 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:34:11,837 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:34:11,838 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:34:11,838 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:34:11,838 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:34:11,838 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:34:11,839 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:34:11,839 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 22:34:12,071 INFO wandb-upload_0:2037873 [upload_job.py:push():130] Uploaded file /tmp/tmpn9pnfdm3wandb/7pij9etm-media/table/eval/success_rate_ci_5_2d447e7da5390d365153.table.json +2025-01-04 22:34:12,182 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:34:12,182 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/wandb-summary.json +2025-01-04 22:34:12,182 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/events.out.tfevents.1736047719.leonmkim-ROG-Strix-G15CS-G15CS.2037421.0 +2025-01-04 22:34:12,182 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/table/eval/total_success_rate_ci_6_157933031db0c923cb36.table.json +2025-01-04 22:34:12,183 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/table/eval +2025-01-04 22:34:12,192 INFO wandb-upload_2:2037873 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpx0hyb_gx +2025-01-04 22:34:12,384 INFO wandb-upload_1:2037873 [upload_job.py:push():130] Uploaded file /tmp/tmpn9pnfdm3wandb/qhw5avsz-media/table/eval/total_success_rate_ci_6_157933031db0c923cb36.table.json +2025-01-04 22:34:12,591 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:12,605 INFO SenderThread:2037873 [sender.py:send_request_log_artifact():1525] logged artifact run-iqjkgl2e-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE5NTk0Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg2MzM5NA==', 'latestArtifact': None}} +2025-01-04 22:34:12,605 DEBUG SenderThread:2037873 [sender.py:send():379] send: files +2025-01-04 22:34:12,605 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_157933031db0c923cb36.table.json with policy now +2025-01-04 22:34:12,605 DEBUG SenderThread:2037873 [sender.py:send():379] send: history +2025-01-04 22:34:12,605 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:34:12,606 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:34:12,832 INFO wandb-upload_0:2037873 [upload_job.py:push():130] Uploaded file /tmp/tmpn9pnfdm3wandb/n4ysvd5d-media/table/eval/total_success_rate_ci_6_157933031db0c923cb36.table.json +2025-01-04 22:34:13,182 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/wandb-summary.json +2025-01-04 22:34:13,591 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:13,606 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:34:14,183 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:34:14,591 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:15,592 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:16,592 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:17,592 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:18,592 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:19,506 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:34:19,592 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:20,592 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:21,592 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:22,592 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:23,593 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:24,494 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 22:34:24,494 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: stop_status +2025-01-04 22:34:24,568 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:34:24,593 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:25,593 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:26,593 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:27,593 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:28,206 DEBUG SenderThread:2037873 [sender.py:send():379] send: exit +2025-01-04 22:34:28,206 INFO SenderThread:2037873 [sender.py:send_exit():586] handling exit code: 0 +2025-01-04 22:34:28,206 INFO SenderThread:2037873 [sender.py:send_exit():588] handling runtime: 349 +2025-01-04 22:34:28,206 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:34:28,207 INFO SenderThread:2037873 [sender.py:send_exit():594] send defer +2025-01-04 22:34:28,207 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:34:28,207 INFO HandlerThread:2037873 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-04 22:34:28,207 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: defer +2025-01-04 22:34:28,207 INFO SenderThread:2037873 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-04 22:34:28,207 INFO SenderThread:2037873 [sender.py:transition_state():614] send defer: 1 +2025-01-04 22:34:28,207 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:34:28,207 INFO HandlerThread:2037873 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-04 22:34:28,207 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: defer +2025-01-04 22:34:28,207 INFO SenderThread:2037873 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-04 22:34:28,207 INFO SenderThread:2037873 [sender.py:transition_state():614] send defer: 2 +2025-01-04 22:34:28,208 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:34:28,208 INFO HandlerThread:2037873 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-04 22:34:28,208 INFO HandlerThread:2037873 [system_monitor.py:finish():203] Stopping system monitor +2025-01-04 22:34:28,208 DEBUG SystemMonitor:2037873 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-04 22:34:28,208 DEBUG SystemMonitor:2037873 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-04 22:34:28,209 INFO HandlerThread:2037873 [interfaces.py:finish():200] Joined cpu monitor +2025-01-04 22:34:28,209 INFO HandlerThread:2037873 [interfaces.py:finish():200] Joined disk monitor +2025-01-04 22:34:28,261 INFO HandlerThread:2037873 [interfaces.py:finish():200] Joined gpu monitor +2025-01-04 22:34:28,262 INFO HandlerThread:2037873 [interfaces.py:finish():200] Joined memory monitor +2025-01-04 22:34:28,262 INFO HandlerThread:2037873 [interfaces.py:finish():200] Joined network monitor +2025-01-04 22:34:28,262 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: defer +2025-01-04 22:34:28,262 INFO SenderThread:2037873 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-04 22:34:28,262 INFO SenderThread:2037873 [sender.py:transition_state():614] send defer: 3 +2025-01-04 22:34:28,262 DEBUG SenderThread:2037873 [sender.py:send():379] send: stats +2025-01-04 22:34:28,263 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:34:28,263 INFO HandlerThread:2037873 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-04 22:34:28,263 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: defer +2025-01-04 22:34:28,263 INFO SenderThread:2037873 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-04 22:34:28,263 INFO SenderThread:2037873 [sender.py:transition_state():614] send defer: 4 +2025-01-04 22:34:28,263 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:34:28,263 INFO HandlerThread:2037873 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-04 22:34:29,189 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/wandb-summary.json +2025-01-04 22:34:30,190 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:34:34,988 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:34:34,988 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:34:34,989 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: keepalive +2025-01-04 22:34:34,990 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: defer +2025-01-04 22:34:34,990 INFO SenderThread:2037873 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-04 22:34:34,990 INFO SenderThread:2037873 [sender.py:transition_state():614] send defer: 5 +2025-01-04 22:34:34,990 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:34:34,990 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:34:34,990 INFO HandlerThread:2037873 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-04 22:34:34,990 DEBUG SenderThread:2037873 [sender.py:send():379] send: history +2025-01-04 22:34:34,991 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: summary_record +2025-01-04 22:34:34,991 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:34:34,992 DEBUG SenderThread:2037873 [sender.py:send():379] send: summary +2025-01-04 22:34:34,995 INFO SenderThread:2037873 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 22:34:34,995 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: defer +2025-01-04 22:34:34,995 INFO SenderThread:2037873 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-04 22:34:34,996 INFO SenderThread:2037873 [sender.py:transition_state():614] send defer: 6 +2025-01-04 22:34:34,996 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:34:34,996 INFO HandlerThread:2037873 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-04 22:34:34,996 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: defer +2025-01-04 22:34:34,996 INFO SenderThread:2037873 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-04 22:34:34,999 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 22:34:35,134 INFO SenderThread:2037873 [sender.py:transition_state():614] send defer: 7 +2025-01-04 22:34:35,134 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:34:35,134 INFO HandlerThread:2037873 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-04 22:34:35,134 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: defer +2025-01-04 22:34:35,135 INFO SenderThread:2037873 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-04 22:34:35,192 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/config.yaml +2025-01-04 22:34:35,192 INFO Thread-12 :2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/wandb-summary.json +2025-01-04 22:34:35,207 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:34:37,751 INFO SenderThread:2037873 [sender.py:transition_state():614] send defer: 8 +2025-01-04 22:34:37,752 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:34:37,752 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:34:37,752 INFO HandlerThread:2037873 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-04 22:34:37,752 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: defer +2025-01-04 22:34:37,752 INFO SenderThread:2037873 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-04 22:34:37,752 INFO SenderThread:2037873 [job_builder.py:build():440] Attempting to build job artifact +2025-01-04 22:34:37,754 INFO SenderThread:2037873 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-04 22:34:37,756 INFO SenderThread:2037873 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-04 22:34:37,770 INFO SenderThread:2037873 [sender.py:transition_state():614] send defer: 9 +2025-01-04 22:34:37,770 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:34:37,770 INFO HandlerThread:2037873 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-04 22:34:37,770 DEBUG SenderThread:2037873 [sender.py:send():379] send: artifact +2025-01-04 22:34:37,982 INFO SenderThread:2037873 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxOTExMTc3MQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTE5MjI0Ng==', 'versionIndex': 420}}} +2025-01-04 22:34:37,982 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: defer +2025-01-04 22:34:37,982 INFO SenderThread:2037873 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-04 22:34:37,982 INFO SenderThread:2037873 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-04 22:34:38,193 INFO SenderThread:2037873 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:34:38,193 INFO SenderThread:2037873 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files +2025-01-04 22:34:38,193 INFO SenderThread:2037873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/config.yaml config.yaml +2025-01-04 22:34:38,193 INFO SenderThread:2037873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/wandb-summary.json wandb-summary.json +2025-01-04 22:34:38,193 INFO SenderThread:2037873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/requirements.txt requirements.txt +2025-01-04 22:34:38,193 INFO SenderThread:2037873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/wandb-metadata.json wandb-metadata.json +2025-01-04 22:34:38,193 INFO SenderThread:2037873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/events.out.tfevents.1736047719.leonmkim-ROG-Strix-G15CS-G15CS.2037421.0 events.out.tfevents.1736047719.leonmkim-ROG-Strix-G15CS-G15CS.2037421.0 +2025-01-04 22:34:38,194 INFO SenderThread:2037873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log output.log +2025-01-04 22:34:38,194 INFO SenderThread:2037873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/diff.patch diff.patch +2025-01-04 22:34:38,198 INFO SenderThread:2037873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-04 22:34:38,199 INFO SenderThread:2037873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval/2_eval_2_fef06f9228f28f137565.mp4 media/videos/eval/2_eval_2_fef06f9228f28f137565.mp4 +2025-01-04 22:34:38,199 INFO SenderThread:2037873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval/4_eval_4_d691d8c6f8564711e47c.mp4 media/videos/eval/4_eval_4_d691d8c6f8564711e47c.mp4 +2025-01-04 22:34:38,199 INFO SenderThread:2037873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval/1_eval_1_fcc2c84034b2f80493e2.mp4 media/videos/eval/1_eval_1_fcc2c84034b2f80493e2.mp4 +2025-01-04 22:34:38,199 INFO SenderThread:2037873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval/0_eval_0_3f829840c9d5231f7c2a.mp4 media/videos/eval/0_eval_0_3f829840c9d5231f7c2a.mp4 +2025-01-04 22:34:38,199 INFO SenderThread:2037873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval/3_eval_3_2ceb841f3e55434b9bc7.mp4 media/videos/eval/3_eval_3_2ceb841f3e55434b9bc7.mp4 +2025-01-04 22:34:38,201 INFO SenderThread:2037873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/table/eval/total_success_rate_ci_6_157933031db0c923cb36.table.json media/table/eval/total_success_rate_ci_6_157933031db0c923cb36.table.json +2025-01-04 22:34:38,201 INFO SenderThread:2037873 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/table/eval/success_rate_ci_5_2d447e7da5390d365153.table.json media/table/eval/success_rate_ci_5_2d447e7da5390d365153.table.json +2025-01-04 22:34:38,201 INFO SenderThread:2037873 [sender.py:transition_state():614] send defer: 10 +2025-01-04 22:34:38,202 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:34:38,202 INFO HandlerThread:2037873 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-04 22:34:38,203 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: defer +2025-01-04 22:34:38,203 INFO SenderThread:2037873 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-04 22:34:38,203 INFO SenderThread:2037873 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 22:34:38,208 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:34:38,208 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:34:38,373 INFO wandb-upload_0:2037873 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/requirements.txt +2025-01-04 22:34:38,387 INFO wandb-upload_1:2037873 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/config.yaml +2025-01-04 22:34:38,395 INFO wandb-upload_2:2037873 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/wandb-summary.json +2025-01-04 22:34:38,446 INFO wandb-upload_4:2037873 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/output.log +2025-01-04 22:34:38,468 INFO wandb-upload_3:2037873 [upload_job.py:push():130] Uploaded file /tmp/tmpn9pnfdm3wandb/vb529jnn-events.out.tfevents.1736047719.leonmkim-ROG-Strix-G15CS-G15CS.2037421.0 +2025-01-04 22:34:38,669 INFO Thread-11 (_thread_body):2037873 [sender.py:transition_state():614] send defer: 11 +2025-01-04 22:34:38,669 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:34:38,669 INFO HandlerThread:2037873 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-04 22:34:38,669 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: defer +2025-01-04 22:34:38,670 INFO SenderThread:2037873 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-04 22:34:38,670 INFO SenderThread:2037873 [file_pusher.py:join():175] waiting for file pusher +2025-01-04 22:34:38,671 INFO SenderThread:2037873 [sender.py:transition_state():614] send defer: 12 +2025-01-04 22:34:38,671 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:34:38,671 INFO HandlerThread:2037873 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-04 22:34:38,671 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: defer +2025-01-04 22:34:38,671 INFO SenderThread:2037873 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-04 22:34:38,671 INFO SenderThread:2037873 [file_stream.py:finish():601] file stream finish called +2025-01-04 22:34:38,741 INFO SenderThread:2037873 [file_stream.py:finish():605] file stream finish is done +2025-01-04 22:34:38,741 INFO SenderThread:2037873 [sender.py:transition_state():614] send defer: 13 +2025-01-04 22:34:38,741 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:34:38,741 INFO HandlerThread:2037873 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-04 22:34:38,741 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: defer +2025-01-04 22:34:38,741 INFO SenderThread:2037873 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-04 22:34:38,741 INFO SenderThread:2037873 [sender.py:transition_state():614] send defer: 14 +2025-01-04 22:34:38,742 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: defer +2025-01-04 22:34:38,742 INFO HandlerThread:2037873 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-04 22:34:38,742 DEBUG SenderThread:2037873 [sender.py:send():379] send: final +2025-01-04 22:34:38,742 DEBUG SenderThread:2037873 [sender.py:send():379] send: footer +2025-01-04 22:34:38,742 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: defer +2025-01-04 22:34:38,742 INFO SenderThread:2037873 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-04 22:34:38,743 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:34:38,743 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:34:38,743 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: server_info +2025-01-04 22:34:38,743 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: server_info +2025-01-04 22:34:38,744 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 22:34:38,745 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: get_summary +2025-01-04 22:34:38,745 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-04 22:34:38,746 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 22:34:38,794 DEBUG SenderThread:2037873 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 22:34:38,794 INFO MainThread:2037873 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-04 22:34:38,794 INFO MainThread:2037873 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-04 22:34:38,794 INFO MainThread:2037873 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-04 22:34:38,795 DEBUG HandlerThread:2037873 [handler.py:handle_request():158] handle_request: shutdown +2025-01-04 22:34:38,795 INFO HandlerThread:2037873 [handler.py:finish():882] shutting down handler +2025-01-04 22:34:39,745 INFO WriterThread:2037873 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/run-iqjkgl2e.wandb +2025-01-04 22:34:39,794 INFO SenderThread:2037873 [sender.py:finish():1615] shutting down sender +2025-01-04 22:34:39,794 INFO SenderThread:2037873 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 22:34:39,794 INFO SenderThread:2037873 [file_pusher.py:join():175] waiting for file pusher diff --git a/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/logs/debug.log b/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..a53a864479784676dd04cd246ef5d2a38fb3315f --- /dev/null +++ b/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-04 22:28:38,817 INFO MainThread:2037421 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-04 22:28:38,817 INFO MainThread:2037421 [wandb_setup.py:_flush():76] Configure stats pid to 2037421 +2025-01-04 22:28:38,817 INFO MainThread:2037421 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-04 22:28:38,817 INFO MainThread:2037421 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/settings +2025-01-04 22:28:38,817 INFO MainThread:2037421 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-04 22:28:38,817 INFO MainThread:2037421 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-04 22:28:38,817 INFO MainThread:2037421 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-04 22:28:38,817 INFO MainThread:2037421 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-04 22:28:38,817 INFO MainThread:2037421 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/logs/debug.log +2025-01-04 22:28:38,817 INFO MainThread:2037421 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/logs/debug-internal.log +2025-01-04 22:28:38,817 INFO MainThread:2037421 [wandb_init.py:init():569] calling init triggers +2025-01-04 22:28:38,817 INFO MainThread:2037421 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1926_1', 'wandb_notes': '1926_1_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e'} +2025-01-04 22:28:38,817 INFO MainThread:2037421 [wandb_init.py:init():619] starting backend +2025-01-04 22:28:38,818 INFO MainThread:2037421 [wandb_init.py:init():623] setting up manager +2025-01-04 22:28:38,822 INFO MainThread:2037421 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-04 22:28:38,822 INFO MainThread:2037421 [wandb_init.py:init():631] backend started and connected +2025-01-04 22:28:38,835 INFO MainThread:2037421 [wandb_init.py:init():720] updated telemetry +2025-01-04 22:28:38,843 INFO MainThread:2037421 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-04 22:28:39,090 INFO MainThread:2037421 [wandb_run.py:_on_init():2435] communicating current version +2025-01-04 22:28:39,131 INFO MainThread:2037421 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-04 22:28:39,132 INFO MainThread:2037421 [wandb_init.py:init():804] starting run threads in backend +2025-01-04 22:28:39,488 INFO MainThread:2037421 [wandb_run.py:_console_start():2413] atexit reg +2025-01-04 22:28:39,488 INFO MainThread:2037421 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-04 22:28:39,488 INFO MainThread:2037421 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-04 22:28:39,489 INFO MainThread:2037421 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-04 22:28:39,490 INFO MainThread:2037421 [wandb_init.py:init():847] run started, returning control to user process +2025-01-04 22:28:39,490 INFO MainThread:2037421 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/tb, True +2025-01-04 22:28:52,929 INFO MainThread:2037421 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2025-01-04 22:34:39,795 WARNING MsgRouterThr:2037421 [router.py:message_loop():77] message_loop has been closed diff --git a/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/run-iqjkgl2e.wandb b/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/run-iqjkgl2e.wandb new file mode 100644 index 0000000000000000000000000000000000000000..8834adab0ae7f01a977834e38b9aeb4c41f153fb Binary files /dev/null and b/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/run-iqjkgl2e.wandb differ diff --git a/nayavtp6/.hydra/config.yaml b/nayavtp6/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5614a531a25d335cfde80ac1396030650a108114 --- /dev/null +++ b/nayavtp6/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1926_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/nayavtp6/.hydra/hydra.yaml b/nayavtp6/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..da3c9f35fb4b531011369de32ac2e67740bab56f --- /dev/null +++ b/nayavtp6/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/nayavtp6/.hydra/overrides.yaml b/nayavtp6/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/nayavtp6/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/nayavtp6/episode_rosbags/aligned_depth_to_color_K.npy b/nayavtp6/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/nayavtp6/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/nayavtp6/episode_rosbags/cam_tf_world.npy b/nayavtp6/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/nayavtp6/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/nayavtp6/episode_rosbags/color_K.npy b/nayavtp6/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/nayavtp6/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/nayavtp6/episode_rosbags/depth_K.npy b/nayavtp6/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/nayavtp6/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/nayavtp6/episode_rosbags/episode_0_2025-01-03-21-16-31.bag b/nayavtp6/episode_rosbags/episode_0_2025-01-03-21-16-31.bag new file mode 100644 index 0000000000000000000000000000000000000000..a8d83248fd7e420f08fc11c2ca01d471aab6c485 --- /dev/null +++ b/nayavtp6/episode_rosbags/episode_0_2025-01-03-21-16-31.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b25d907cb4ea75b854cde6c51e11ccc05f0269d4c81ffece34819866bb49a0d2 +size 851801842 diff --git a/nayavtp6/episode_rosbags/episode_1_2025-01-03-21-17-13.bag b/nayavtp6/episode_rosbags/episode_1_2025-01-03-21-17-13.bag new file mode 100644 index 0000000000000000000000000000000000000000..cd4a35a5464ad941955368873ca22af3460118eb --- /dev/null +++ b/nayavtp6/episode_rosbags/episode_1_2025-01-03-21-17-13.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35716a92ac6754e935de4d68cca7afdfcf423173546f11347b94b4e9aaf17901 +size 856861393 diff --git a/nayavtp6/episode_rosbags/episode_2_2025-01-03-21-17-55.bag b/nayavtp6/episode_rosbags/episode_2_2025-01-03-21-17-55.bag new file mode 100644 index 0000000000000000000000000000000000000000..0c7f40eb7c63e7da2cd72e164e15ffa26dd8b509 --- /dev/null +++ b/nayavtp6/episode_rosbags/episode_2_2025-01-03-21-17-55.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9bdd38903059f80636854a7df7bddbef069c907c0eacd725599a3aeadea9f4b1 +size 864009731 diff --git a/nayavtp6/episode_rosbags/episode_3_2025-01-03-21-18-32.bag b/nayavtp6/episode_rosbags/episode_3_2025-01-03-21-18-32.bag new file mode 100644 index 0000000000000000000000000000000000000000..6a1ca86fd65f95a5ead48ce91aa6dfceedc55947 --- /dev/null +++ b/nayavtp6/episode_rosbags/episode_3_2025-01-03-21-18-32.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:165358a05ec02dd2769c380063146eb12974fefbce279bb4327855a8e8059733 +size 1608212915 diff --git a/nayavtp6/episode_rosbags/episode_4_2025-01-03-21-19-19.bag b/nayavtp6/episode_rosbags/episode_4_2025-01-03-21-19-19.bag new file mode 100644 index 0000000000000000000000000000000000000000..68a9ccf982df661b9982e971695cb0ad0a39412e --- /dev/null +++ b/nayavtp6/episode_rosbags/episode_4_2025-01-03-21-19-19.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5906a4771103571b837db5a8c6d8488981da20836370f725de9531cfcbcb4ea4 +size 1277692796 diff --git a/nayavtp6/eval_robot.log b/nayavtp6/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..aa29480dc5c487573b5d32917d73e2fbaa4d4763 --- /dev/null +++ b/nayavtp6/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-03 21:16:00,422][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-03 21:16:00,425][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/nayavtp6/eval_video/0_eval.mp4 b/nayavtp6/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..438ef780f399c137026ac497af9eda8a3d3292e7 --- /dev/null +++ b/nayavtp6/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29e1a738e40422f191005ecb6090c8f1a958b08c29ad80f4472e2a4a928eded4 +size 565195 diff --git a/nayavtp6/eval_video/1_eval.mp4 b/nayavtp6/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7ae1d8c194a1bc40505eb24ab6344f3138f1ea99 --- /dev/null +++ b/nayavtp6/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91c0aab1f5aa1b260cf6cc217e49c394fc6b03957edac0e7c36256b102aa7edb +size 561687 diff --git a/nayavtp6/eval_video/2_eval.mp4 b/nayavtp6/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b8e220ecbb4960726d7193c984be04a4e5b164de --- /dev/null +++ b/nayavtp6/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a653d18806cfc6db0e07b96db57207d1ae0bb118ab46768e8215693a45563227 +size 581960 diff --git a/nayavtp6/eval_video/3_eval.mp4 b/nayavtp6/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0cb40052c3bfc42b0b0d5dfb98cf4002976d9cb4 --- /dev/null +++ b/nayavtp6/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a145a501509e034700f1d2965bd668794ffea38d3be9b9defb32ea120f938937 +size 966517 diff --git a/nayavtp6/eval_video/4_eval.mp4 b/nayavtp6/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b24ec897e8c2e93c55d183567ff7ba0a58e12c67 --- /dev/null +++ b/nayavtp6/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ec81cfff6351c2b5db45e5552d306f19b1d1dd60a84948557b9ee24dc8bec45 +size 789769 diff --git a/nayavtp6/tb/events.out.tfevents.1735956966.leonmkim-ROG-Strix-G15CS-G15CS.1975301.0 b/nayavtp6/tb/events.out.tfevents.1735956966.leonmkim-ROG-Strix-G15CS-G15CS.1975301.0 new file mode 100644 index 0000000000000000000000000000000000000000..9eb156dca4ba2ce9ba4ac3f46fd638708baf8d41 --- /dev/null +++ b/nayavtp6/tb/events.out.tfevents.1735956966.leonmkim-ROG-Strix-G15CS-G15CS.1975301.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c95bb0278fc95bb24fc8ac2456a394b7f0c2a76eac3b8a21ac087b4c4c1b8fc1 +size 6499 diff --git a/nayavtp6/wandb/debug-internal.log b/nayavtp6/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..7ff7ccd4d26f8860de957ab8104917580becb96d --- /dev/null +++ b/nayavtp6/wandb/debug-internal.log @@ -0,0 +1,738 @@ +2025-01-03 21:16:06,131 INFO StreamThr :1975487 [internal.py:wandb_internal():85] W&B internal server running at pid: 1975487, started at: 2025-01-03 21:16:06.130701 +2025-01-03 21:16:06,132 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status +2025-01-03 21:16:06,152 INFO WriterThread:1975487 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/run-nayavtp6.wandb +2025-01-03 21:16:06,152 DEBUG SenderThread:1975487 [sender.py:send():379] send: header +2025-01-03 21:16:06,160 DEBUG SenderThread:1975487 [sender.py:send():379] send: run +2025-01-03 21:16:06,394 INFO SenderThread:1975487 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files +2025-01-03 21:16:06,395 INFO SenderThread:1975487 [sender.py:_start_run_threads():1188] run started: nayavtp6 with start time 1735956966.13103 +2025-01-03 21:16:06,413 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: check_version +2025-01-03 21:16:06,413 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: check_version +2025-01-03 21:16:06,492 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: run_start +2025-01-03 21:16:06,496 DEBUG HandlerThread:1975487 [system_info.py:__init__():26] System info init +2025-01-03 21:16:06,497 DEBUG HandlerThread:1975487 [system_info.py:__init__():41] System info init done +2025-01-03 21:16:06,497 INFO HandlerThread:1975487 [system_monitor.py:start():194] Starting system monitor +2025-01-03 21:16:06,497 INFO SystemMonitor:1975487 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-03 21:16:06,497 INFO HandlerThread:1975487 [system_monitor.py:probe():214] Collecting system info +2025-01-03 21:16:06,497 INFO SystemMonitor:1975487 [interfaces.py:start():188] Started cpu monitoring +2025-01-03 21:16:06,498 INFO SystemMonitor:1975487 [interfaces.py:start():188] Started disk monitoring +2025-01-03 21:16:06,498 INFO SystemMonitor:1975487 [interfaces.py:start():188] Started gpu monitoring +2025-01-03 21:16:06,499 INFO SystemMonitor:1975487 [interfaces.py:start():188] Started memory monitoring +2025-01-03 21:16:06,499 INFO SystemMonitor:1975487 [interfaces.py:start():188] Started network monitoring +2025-01-03 21:16:06,509 DEBUG HandlerThread:1975487 [system_info.py:probe():152] Probing system +2025-01-03 21:16:06,511 DEBUG HandlerThread:1975487 [system_info.py:_probe_git():137] Probing git +2025-01-03 21:16:06,515 DEBUG HandlerThread:1975487 [system_info.py:_probe_git():145] Probing git done +2025-01-03 21:16:06,516 DEBUG HandlerThread:1975487 [system_info.py:probe():200] Probing system done +2025-01-03 21:16:06,516 DEBUG HandlerThread:1975487 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-04T02:16:06.510029', 'startedAt': '2025-01-04T02:16:06.123552', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'c537eb6734155592809c2a7db5cca971ec186d72'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.2645, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.999, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.985, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 807.209831237793}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-03 21:16:06,516 INFO HandlerThread:1975487 [system_monitor.py:probe():224] Finished collecting system info +2025-01-03 21:16:06,516 INFO HandlerThread:1975487 [system_monitor.py:probe():227] Publishing system info +2025-01-03 21:16:06,516 DEBUG HandlerThread:1975487 [system_info.py:_save_code():44] Saving code +2025-01-03 21:16:06,517 DEBUG HandlerThread:1975487 [system_info.py:_save_code():67] Saving code done +2025-01-03 21:16:06,517 DEBUG HandlerThread:1975487 [system_info.py:_save_patches():84] Saving git patches +2025-01-03 21:16:06,579 DEBUG HandlerThread:1975487 [system_info.py:_save_patches():126] Saving git patches done +2025-01-03 21:16:06,579 INFO HandlerThread:1975487 [system_monitor.py:probe():229] Finished publishing system info +2025-01-03 21:16:06,583 DEBUG SenderThread:1975487 [sender.py:send():379] send: files +2025-01-03 21:16:06,583 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-03 21:16:06,583 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-03 21:16:06,583 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-03 21:16:06,831 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: python_packages +2025-01-03 21:16:06,831 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: python_packages +2025-01-03 21:16:06,831 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:16:06,832 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:06,832 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:16:06,833 INFO HandlerThread:1975487 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/tb" + save: true +} +control { + relay_id: "729b5f9de7ed46d2a8965422843bb4ce" +} +_info { + stream_id: "nayavtp6" +} + +2025-01-03 21:16:06,833 DEBUG HandlerThread:1975487 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-03 21:16:06,845 INFO wandb-upload_0:1975487 [upload_job.py:push():130] Uploaded file /tmp/tmpo3rn3d09wandb/oxpgz1fy-wandb-metadata.json +2025-01-03 21:16:06,854 INFO wandb-upload_1:1975487 [upload_job.py:push():130] Uploaded file /tmp/tmpo3rn3d09wandb/2wylazit-code/FISH/eval_robot.py +2025-01-03 21:16:06,877 INFO wandb-upload_2:1975487 [upload_job.py:push():130] Uploaded file /tmp/tmpo3rn3d09wandb/jcr2f0l8-diff.patch +2025-01-03 21:16:06,938 DEBUG SenderThread:1975487 [sender.py:send():379] send: telemetry +2025-01-03 21:16:06,938 DEBUG SenderThread:1975487 [sender.py:send():379] send: tbrecord +2025-01-03 21:16:06,939 DEBUG SenderThread:1975487 [sender.py:send():379] send: files +2025-01-03 21:16:06,939 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file events.out.tfevents.1735956966.leonmkim-ROG-Strix-G15CS-G15CS.1975301.0 with policy live +2025-01-03 21:16:07,192 INFO wandb-upload_0:1975487 [upload_job.py:push():130] Uploaded file /tmp/tmpo3rn3d09wandb/ehlc0r0c-events.out.tfevents.1735956966.leonmkim-ROG-Strix-G15CS-G15CS.1975301.0 +2025-01-03 21:16:07,396 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/wandb-metadata.json +2025-01-03 21:16:07,396 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/requirements.txt +2025-01-03 21:16:07,396 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/code/FISH/eval_robot.py +2025-01-03 21:16:07,397 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/events.out.tfevents.1735956966.leonmkim-ROG-Strix-G15CS-G15CS.1975301.0 +2025-01-03 21:16:07,397 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/diff.patch +2025-01-03 21:16:07,397 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:16:07,397 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/code/FISH +2025-01-03 21:16:07,397 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/code +2025-01-03 21:16:07,831 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:08,831 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:09,395 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:16:09,832 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:10,832 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:11,356 DEBUG SenderThread:1975487 [sender.py:send():379] send: config +2025-01-03 21:16:11,356 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:16:11,832 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:12,832 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:13,397 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:16:13,832 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:14,832 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:15,833 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:16,832 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:16:16,833 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:17,398 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:16:17,833 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:18,841 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:19,834 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:20,834 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:21,399 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:16:21,831 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:16:21,831 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:16:21,873 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:21,952 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:16:22,834 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:23,834 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:24,834 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:25,835 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:26,835 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:27,833 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:16:27,835 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:28,835 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:29,401 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:16:29,835 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:30,835 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:31,402 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:16:31,835 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:32,835 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:32,971 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:16:33,402 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:16:33,836 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:34,836 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:35,836 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:36,831 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:16:36,831 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:16:36,873 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:37,836 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:38,836 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:38,936 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:16:39,404 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/config.yaml +2025-01-03 21:16:39,836 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:40,836 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:41,836 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:42,836 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:43,837 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:44,168 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:16:44,422 DEBUG SenderThread:1975487 [sender.py:send():379] send: files +2025-01-03 21:16:44,422 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_29e1a738e40422f19100.mp4 with policy now +2025-01-03 21:16:44,423 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:16:44,425 DEBUG SenderThread:1975487 [sender.py:send():379] send: history +2025-01-03 21:16:44,425 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:16:44,426 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:16:44,696 INFO wandb-upload_2:1975487 [upload_job.py:push():130] Uploaded file /tmp/tmpo3rn3d09wandb/euxkfv4q-media/videos/eval/0_eval_0_29e1a738e40422f19100.mp4 +2025-01-03 21:16:44,837 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:45,406 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:16:45,406 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval/0_eval_0_29e1a738e40422f19100.mp4 +2025-01-03 21:16:45,406 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/wandb-summary.json +2025-01-03 21:16:45,406 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval +2025-01-03 21:16:45,406 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos +2025-01-03 21:16:45,406 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media +2025-01-03 21:16:45,684 INFO wandb-upload_0:1975487 [upload_job.py:push():130] Uploaded file /tmp/tmpo3rn3d09wandb/wxyv8isk-media/videos/eval/0_eval_0_29e1a738e40422f19100.mp4 +2025-01-03 21:16:45,837 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:46,837 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:47,838 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:48,837 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:49,407 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:16:49,838 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:49,989 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:16:50,838 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:51,831 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:16:51,831 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:16:51,873 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:52,838 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:53,408 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:16:53,838 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:54,838 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:55,838 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:55,939 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:16:56,838 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:57,839 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:58,839 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:59,839 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:00,839 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:01,833 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:17:01,839 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:02,839 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:03,839 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:04,839 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:05,839 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:06,499 DEBUG SystemMonitor:1975487 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-03 21:17:06,500 DEBUG SenderThread:1975487 [sender.py:send():379] send: stats +2025-01-03 21:17:06,831 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:17:06,832 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:17:06,873 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:06,937 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:17:07,840 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:08,840 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:09,414 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/events.out.tfevents.1735956966.leonmkim-ROG-Strix-G15CS-G15CS.1975301.0 +2025-01-03 21:17:09,661 INFO wandb-upload_2:1975487 [upload_job.py:push():130] Uploaded file /tmp/tmpo3rn3d09wandb/5yur69kf-events.out.tfevents.1735956966.leonmkim-ROG-Strix-G15CS-G15CS.1975301.0 +2025-01-03 21:17:09,840 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:10,840 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:11,414 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:17:11,840 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:12,834 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:17:12,840 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:13,840 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:14,840 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:15,416 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:17:15,841 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:16,841 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:17,834 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:17:17,841 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:18,841 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:19,841 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:20,841 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:21,832 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:17:21,832 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:17:21,873 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:22,841 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:22,891 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:17:23,842 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:24,853 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:25,853 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:25,965 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:17:25,965 DEBUG SenderThread:1975487 [sender.py:send():379] send: files +2025-01-03 21:17:25,965 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_91c0aab1f5aa1b260cf6.mp4 with policy now +2025-01-03 21:17:25,975 DEBUG SenderThread:1975487 [sender.py:send():379] send: history +2025-01-03 21:17:25,975 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:17:25,975 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:17:26,238 INFO wandb-upload_0:1975487 [upload_job.py:push():130] Uploaded file /tmp/tmpo3rn3d09wandb/pfplswdq-media/videos/eval/1_eval_1_91c0aab1f5aa1b260cf6.mp4 +2025-01-03 21:17:26,420 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/wandb-summary.json +2025-01-03 21:17:26,420 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval/1_eval_1_91c0aab1f5aa1b260cf6.mp4 +2025-01-03 21:17:26,420 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval +2025-01-03 21:17:26,678 INFO wandb-upload_2:1975487 [upload_job.py:push():130] Uploaded file /tmp/tmpo3rn3d09wandb/d6kwnv5m-media/videos/eval/1_eval_1_91c0aab1f5aa1b260cf6.mp4 +2025-01-03 21:17:26,847 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:27,420 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:17:27,847 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:28,844 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:17:28,850 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:29,848 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:30,849 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:31,421 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:17:31,850 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:32,846 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:33,422 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:17:33,846 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:34,441 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:17:34,848 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:35,846 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:36,500 DEBUG SenderThread:1975487 [sender.py:send():379] send: stats +2025-01-03 21:17:36,832 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:17:36,832 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:17:36,873 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:37,847 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:38,847 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:39,847 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:39,982 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:17:40,847 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:41,850 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:42,849 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:43,848 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:44,848 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:45,836 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:17:45,848 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:46,854 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:47,849 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:48,849 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:49,851 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:50,836 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:17:50,851 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:51,833 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:17:51,833 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:17:51,873 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:52,232 DEBUG SenderThread:1975487 [sender.py:send():379] send: history +2025-01-03 21:17:52,232 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:17:52,234 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:17:52,428 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/wandb-summary.json +2025-01-03 21:17:52,428 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/events.out.tfevents.1735956966.leonmkim-ROG-Strix-G15CS-G15CS.1975301.0 +2025-01-03 21:17:52,853 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:53,429 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:17:53,852 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:54,852 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:55,429 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:17:55,857 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:55,863 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:17:56,855 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:57,430 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:17:57,855 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:58,855 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:59,855 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:00,855 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:01,836 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:18:01,856 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:02,856 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:03,856 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:04,856 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:05,856 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:06,501 DEBUG SenderThread:1975487 [sender.py:send():379] send: stats +2025-01-03 21:18:06,832 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:18:06,832 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:18:06,877 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:06,911 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:18:07,857 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:08,704 DEBUG SenderThread:1975487 [sender.py:send():379] send: files +2025-01-03 21:18:08,704 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_a653d18806cfc6db0e07.mp4 with policy now +2025-01-03 21:18:08,709 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:18:08,709 DEBUG SenderThread:1975487 [sender.py:send():379] send: history +2025-01-03 21:18:08,709 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:18:08,710 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:18:08,857 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:08,950 INFO wandb-upload_0:1975487 [upload_job.py:push():130] Uploaded file /tmp/tmpo3rn3d09wandb/dyha6n7f-media/videos/eval/2_eval_2_a653d18806cfc6db0e07.mp4 +2025-01-03 21:18:09,434 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/wandb-summary.json +2025-01-03 21:18:09,434 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:18:09,434 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval/2_eval_2_a653d18806cfc6db0e07.mp4 +2025-01-03 21:18:09,434 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval +2025-01-03 21:18:09,698 INFO wandb-upload_2:1975487 [upload_job.py:push():130] Uploaded file /tmp/tmpo3rn3d09wandb/9rq9zg5h-media/videos/eval/2_eval_2_a653d18806cfc6db0e07.mp4 +2025-01-03 21:18:09,869 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:10,872 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:11,435 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:18:11,869 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:12,139 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:18:12,872 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:13,435 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:18:13,869 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:14,870 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:15,872 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:16,874 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:17,437 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:18:17,850 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:18:17,878 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:18,873 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:19,874 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:20,877 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:21,835 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:18:21,836 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:18:21,877 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:22,877 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:22,969 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:18:23,875 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:24,878 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:25,875 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:26,882 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:27,879 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:28,730 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:18:28,877 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:29,057 DEBUG SenderThread:1975487 [sender.py:send():379] send: history +2025-01-03 21:18:29,057 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:18:29,058 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:18:29,441 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/wandb-summary.json +2025-01-03 21:18:29,441 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/events.out.tfevents.1735956966.leonmkim-ROG-Strix-G15CS-G15CS.1975301.0 +2025-01-03 21:18:29,676 INFO wandb-upload_0:1975487 [upload_job.py:push():130] Uploaded file /tmp/tmpo3rn3d09wandb/gutyp7n3-events.out.tfevents.1735956966.leonmkim-ROG-Strix-G15CS-G15CS.1975301.0 +2025-01-03 21:18:29,883 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:30,888 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:31,441 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:18:31,877 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:32,877 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:33,878 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:34,111 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:18:34,878 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:35,443 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:18:35,878 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:36,501 DEBUG SenderThread:1975487 [sender.py:send():379] send: stats +2025-01-03 21:18:36,833 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:18:36,833 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:18:36,878 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:37,878 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:38,878 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:39,444 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:18:39,801 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:18:39,878 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:40,878 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:41,880 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:42,880 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:43,446 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:18:43,880 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:44,881 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:45,489 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:18:45,881 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:46,881 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:47,883 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:48,882 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:49,884 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:50,847 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:18:50,884 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:51,833 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:18:51,833 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:18:51,884 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:52,884 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:53,884 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:54,886 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:55,450 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:18:55,887 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:18:55,889 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:55,953 DEBUG SenderThread:1975487 [sender.py:send():379] send: files +2025-01-03 21:18:55,953 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_a145a501509e034700f1.mp4 with policy now +2025-01-03 21:18:55,957 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:18:55,957 DEBUG SenderThread:1975487 [sender.py:send():379] send: history +2025-01-03 21:18:55,957 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:18:55,958 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:18:56,210 INFO wandb-upload_2:1975487 [upload_job.py:push():130] Uploaded file /tmp/tmpo3rn3d09wandb/1jybtjn8-media/videos/eval/3_eval_3_a145a501509e034700f1.mp4 +2025-01-03 21:18:56,450 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/wandb-summary.json +2025-01-03 21:18:56,450 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval/3_eval_3_a145a501509e034700f1.mp4 +2025-01-03 21:18:56,450 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval +2025-01-03 21:18:56,708 INFO wandb-upload_0:1975487 [upload_job.py:push():130] Uploaded file /tmp/tmpo3rn3d09wandb/amwdcsax-media/videos/eval/3_eval_3_a145a501509e034700f1.mp4 +2025-01-03 21:18:56,889 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:57,450 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:18:57,892 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:58,902 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:59,898 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:00,905 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:01,452 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:19:01,637 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:19:01,904 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:02,899 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:03,452 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:19:03,901 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:04,901 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:05,904 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:06,502 DEBUG SenderThread:1975487 [sender.py:send():379] send: stats +2025-01-03 21:19:06,835 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:19:06,835 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:19:06,915 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:06,991 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:19:07,916 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:08,908 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:09,910 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:10,903 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:11,916 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:12,859 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:19:12,914 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:13,912 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:14,913 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:15,912 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:16,112 DEBUG SenderThread:1975487 [sender.py:send():379] send: history +2025-01-03 21:19:16,112 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:19:16,113 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:19:16,457 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/wandb-summary.json +2025-01-03 21:19:16,457 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/events.out.tfevents.1735956966.leonmkim-ROG-Strix-G15CS-G15CS.1975301.0 +2025-01-03 21:19:16,920 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:17,457 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:19:17,913 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:18,858 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:19:18,917 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:19,458 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:19:19,911 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:20,911 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:21,459 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:19:21,833 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:19:21,834 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:19:21,911 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:22,911 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:23,893 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:19:23,912 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:24,912 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:25,912 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:26,912 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:27,912 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:28,912 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:29,853 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:19:29,913 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:30,912 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:31,913 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:32,913 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:33,913 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:34,915 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:35,463 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:19:35,677 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:19:35,921 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:36,502 DEBUG SenderThread:1975487 [sender.py:send():379] send: stats +2025-01-03 21:19:36,833 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:19:36,834 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:19:36,913 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:37,464 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:19:37,918 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:38,256 DEBUG SenderThread:1975487 [sender.py:send():379] send: files +2025-01-03 21:19:38,256 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_8ec81cfff6351c2b5db4.mp4 with policy now +2025-01-03 21:19:38,260 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:19:38,261 DEBUG SenderThread:1975487 [sender.py:send():379] send: history +2025-01-03 21:19:38,261 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:19:38,261 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:19:38,465 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/wandb-summary.json +2025-01-03 21:19:38,465 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval/4_eval_4_8ec81cfff6351c2b5db4.mp4 +2025-01-03 21:19:38,465 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval +2025-01-03 21:19:38,500 INFO wandb-upload_2:1975487 [upload_job.py:push():130] Uploaded file /tmp/tmpo3rn3d09wandb/g9puoe08-media/videos/eval/4_eval_4_8ec81cfff6351c2b5db4.mp4 +2025-01-03 21:19:38,752 INFO wandb-upload_0:1975487 [upload_job.py:push():130] Uploaded file /tmp/tmpo3rn3d09wandb/e8pp42sj-media/videos/eval/4_eval_4_8ec81cfff6351c2b5db4.mp4 +2025-01-03 21:19:38,920 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:39,465 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:19:39,921 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:40,933 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:41,337 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:19:41,466 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:19:41,928 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:42,939 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:43,466 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:19:43,934 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:44,941 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:45,940 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:46,865 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:19:46,958 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:47,468 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:19:47,955 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:48,947 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:49,951 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:50,952 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:51,835 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:19:51,836 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:19:51,902 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:19:51,956 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:52,955 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:53,958 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:54,959 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:55,959 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:56,959 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:57,861 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:19:57,961 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:58,958 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:59,966 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:00,961 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:01,959 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:02,880 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:20:02,979 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:03,980 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:04,965 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:05,029 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 21:20:05,030 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 21:20:05,189 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:20:05,474 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/events.out.tfevents.1735956966.leonmkim-ROG-Strix-G15CS-G15CS.1975301.0 +2025-01-03 21:20:05,474 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:20:05,474 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-03 21:20:05,474 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media +2025-01-03 21:20:05,474 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/table +2025-01-03 21:20:05,474 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/table/eval +2025-01-03 21:20:05,554 INFO wandb-upload_2:1975487 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpw9iems78 +2025-01-03 21:20:05,593 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 21:20:05,602 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:20:05,652 INFO wandb-upload_1:1975487 [upload_job.py:push():130] Uploaded file /tmp/tmpo3rn3d09wandb/gfkii59r-media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-03 21:20:05,755 INFO wandb-upload_0:1975487 [upload_job.py:push():130] Uploaded file /tmp/tmpo3rn3d09wandb/w7asjmes-events.out.tfevents.1735956966.leonmkim-ROG-Strix-G15CS-G15CS.1975301.0 +2025-01-03 21:20:05,960 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:06,034 INFO SenderThread:1975487 [sender.py:send_request_log_artifact():1525] logged artifact run-nayavtp6-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzkwNDMwNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY3ODMwNg==', 'latestArtifact': None}} +2025-01-03 21:20:06,034 DEBUG SenderThread:1975487 [sender.py:send():379] send: files +2025-01-03 21:20:06,034 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json with policy now +2025-01-03 21:20:06,034 DEBUG SenderThread:1975487 [sender.py:send():379] send: history +2025-01-03 21:20:06,034 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:20:06,035 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:20:06,035 DEBUG SenderThread:1975487 [sender.py:send():379] send: history +2025-01-03 21:20:06,036 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:20:06,036 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:20:06,036 DEBUG SenderThread:1975487 [sender.py:send():379] send: telemetry +2025-01-03 21:20:06,036 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:20:06,038 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:20:06,038 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:20:06,038 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:20:06,038 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:20:06,039 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:20:06,039 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:20:06,039 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:20:06,039 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:20:06,040 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:20:06,040 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:20:06,040 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:20:06,040 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:20:06,040 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:20:06,041 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:20:06,041 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:20:06,041 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:20:06,041 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:20:06,042 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:20:06,042 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:20:06,042 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:20:06,042 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:20:06,043 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 21:20:06,309 INFO wandb-upload_2:1975487 [upload_job.py:push():130] Uploaded file /tmp/tmpo3rn3d09wandb/575q2s1s-media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-03 21:20:06,422 INFO wandb-upload_0:1975487 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpbc37cpza +2025-01-03 21:20:06,474 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/wandb-summary.json +2025-01-03 21:20:06,474 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/events.out.tfevents.1735956966.leonmkim-ROG-Strix-G15CS-G15CS.1975301.0 +2025-01-03 21:20:06,474 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/table/eval/total_success_rate_ci_6_afac0fa2248ad7b10763.table.json +2025-01-03 21:20:06,475 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/table/eval +2025-01-03 21:20:06,834 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:20:06,844 INFO SenderThread:1975487 [sender.py:send_request_log_artifact():1525] logged artifact run-nayavtp6-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzkwNDMxNQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY3ODMwNw==', 'latestArtifact': None}} +2025-01-03 21:20:06,844 DEBUG SenderThread:1975487 [sender.py:send():379] send: files +2025-01-03 21:20:06,844 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_afac0fa2248ad7b10763.table.json with policy now +2025-01-03 21:20:06,845 DEBUG SenderThread:1975487 [sender.py:send():379] send: history +2025-01-03 21:20:06,845 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:20:06,845 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:20:06,845 DEBUG SenderThread:1975487 [sender.py:send():379] send: stats +2025-01-03 21:20:06,846 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:20:06,961 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:07,025 INFO wandb-upload_1:1975487 [upload_job.py:push():130] Uploaded file /tmp/tmpo3rn3d09wandb/0eq29wsn-media/table/eval/total_success_rate_ci_6_afac0fa2248ad7b10763.table.json +2025-01-03 21:20:07,279 INFO wandb-upload_0:1975487 [upload_job.py:push():130] Uploaded file /tmp/tmpo3rn3d09wandb/57fy6k8g-media/table/eval/total_success_rate_ci_6_afac0fa2248ad7b10763.table.json +2025-01-03 21:20:07,475 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/wandb-summary.json +2025-01-03 21:20:07,475 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:20:07,961 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:07,964 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:20:08,961 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:09,961 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:10,289 DEBUG SenderThread:1975487 [sender.py:send():379] send: exit +2025-01-03 21:20:10,289 INFO SenderThread:1975487 [sender.py:send_exit():586] handling exit code: 0 +2025-01-03 21:20:10,290 INFO SenderThread:1975487 [sender.py:send_exit():588] handling runtime: 243 +2025-01-03 21:20:10,290 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:20:10,290 INFO SenderThread:1975487 [sender.py:send_exit():594] send defer +2025-01-03 21:20:10,290 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:20:10,290 INFO HandlerThread:1975487 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-03 21:20:10,290 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: defer +2025-01-03 21:20:10,290 INFO SenderThread:1975487 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-03 21:20:10,290 INFO SenderThread:1975487 [sender.py:transition_state():614] send defer: 1 +2025-01-03 21:20:10,291 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:20:10,291 INFO HandlerThread:1975487 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-03 21:20:10,291 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: defer +2025-01-03 21:20:10,291 INFO SenderThread:1975487 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-03 21:20:10,291 INFO SenderThread:1975487 [sender.py:transition_state():614] send defer: 2 +2025-01-03 21:20:10,291 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:20:10,291 INFO HandlerThread:1975487 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-03 21:20:10,291 INFO HandlerThread:1975487 [system_monitor.py:finish():203] Stopping system monitor +2025-01-03 21:20:10,291 INFO HandlerThread:1975487 [interfaces.py:finish():200] Joined cpu monitor +2025-01-03 21:20:10,292 INFO HandlerThread:1975487 [interfaces.py:finish():200] Joined disk monitor +2025-01-03 21:20:10,292 DEBUG SystemMonitor:1975487 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-03 21:20:10,292 DEBUG SystemMonitor:1975487 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-03 21:20:10,344 INFO HandlerThread:1975487 [interfaces.py:finish():200] Joined gpu monitor +2025-01-03 21:20:10,345 INFO HandlerThread:1975487 [interfaces.py:finish():200] Joined memory monitor +2025-01-03 21:20:10,345 INFO HandlerThread:1975487 [interfaces.py:finish():200] Joined network monitor +2025-01-03 21:20:10,345 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: defer +2025-01-03 21:20:10,345 INFO SenderThread:1975487 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-03 21:20:10,345 INFO SenderThread:1975487 [sender.py:transition_state():614] send defer: 3 +2025-01-03 21:20:10,345 DEBUG SenderThread:1975487 [sender.py:send():379] send: stats +2025-01-03 21:20:10,345 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:20:10,346 INFO HandlerThread:1975487 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-03 21:20:10,346 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: defer +2025-01-03 21:20:10,346 INFO SenderThread:1975487 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-03 21:20:10,346 INFO SenderThread:1975487 [sender.py:transition_state():614] send defer: 4 +2025-01-03 21:20:10,346 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:20:10,346 INFO HandlerThread:1975487 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-03 21:20:10,476 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/wandb-summary.json +2025-01-03 21:20:11,476 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:20:17,174 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:20:17,174 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:20:17,174 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 21:20:17,175 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: defer +2025-01-03 21:20:17,175 INFO SenderThread:1975487 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-03 21:20:17,175 INFO SenderThread:1975487 [sender.py:transition_state():614] send defer: 5 +2025-01-03 21:20:17,175 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:20:17,175 DEBUG SenderThread:1975487 [sender.py:send():379] send: history +2025-01-03 21:20:17,176 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:20:17,176 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:20:17,176 INFO HandlerThread:1975487 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-03 21:20:17,177 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:20:17,177 DEBUG SenderThread:1975487 [sender.py:send():379] send: summary +2025-01-03 21:20:17,177 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:20:17,177 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: defer +2025-01-03 21:20:17,177 INFO SenderThread:1975487 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-03 21:20:17,177 INFO SenderThread:1975487 [sender.py:transition_state():614] send defer: 6 +2025-01-03 21:20:17,177 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:20:17,178 INFO HandlerThread:1975487 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-03 21:20:17,178 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: defer +2025-01-03 21:20:17,178 INFO SenderThread:1975487 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-03 21:20:17,181 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:20:17,296 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:20:17,315 INFO SenderThread:1975487 [sender.py:transition_state():614] send defer: 7 +2025-01-03 21:20:17,315 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:20:17,315 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:20:17,315 INFO HandlerThread:1975487 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-03 21:20:17,315 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: defer +2025-01-03 21:20:17,315 INFO SenderThread:1975487 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-03 21:20:17,479 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/config.yaml +2025-01-03 21:20:17,479 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/wandb-summary.json +2025-01-03 21:20:18,296 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:20:20,712 INFO SenderThread:1975487 [sender.py:transition_state():614] send defer: 8 +2025-01-03 21:20:20,712 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:20:20,712 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:20:20,712 INFO HandlerThread:1975487 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-03 21:20:20,712 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: defer +2025-01-03 21:20:20,712 INFO SenderThread:1975487 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-03 21:20:20,712 INFO SenderThread:1975487 [job_builder.py:build():440] Attempting to build job artifact +2025-01-03 21:20:20,715 INFO SenderThread:1975487 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-03 21:20:20,717 INFO SenderThread:1975487 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-03 21:20:20,729 INFO SenderThread:1975487 [sender.py:transition_state():614] send defer: 9 +2025-01-03 21:20:20,729 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:20:20,729 INFO HandlerThread:1975487 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-03 21:20:20,729 DEBUG SenderThread:1975487 [sender.py:send():379] send: artifact +2025-01-03 21:20:21,150 INFO wandb-upload_2:1975487 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpkky9i0ng +2025-01-03 21:20:21,151 INFO wandb-upload_1:1975487 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpideqz279 +2025-01-03 21:20:21,297 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:20:21,371 INFO wandb-upload_0:1975487 [upload_job.py:push():88] Uploaded file /tmp/tmp5cg23i_h/wandb-job.json +2025-01-03 21:20:21,480 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:20:21,942 INFO SenderThread:1975487 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNzkwNDU3OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNzg5OTM1NQ==', 'versionIndex': 409}}} +2025-01-03 21:20:21,942 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: defer +2025-01-03 21:20:21,942 INFO SenderThread:1975487 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-03 21:20:21,942 INFO SenderThread:1975487 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-03 21:20:22,481 INFO SenderThread:1975487 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files +2025-01-03 21:20:22,481 INFO SenderThread:1975487 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/config.yaml config.yaml +2025-01-03 21:20:22,481 INFO SenderThread:1975487 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/wandb-summary.json wandb-summary.json +2025-01-03 21:20:22,482 INFO SenderThread:1975487 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/requirements.txt requirements.txt +2025-01-03 21:20:22,482 INFO SenderThread:1975487 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/wandb-metadata.json wandb-metadata.json +2025-01-03 21:20:22,482 INFO SenderThread:1975487 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/events.out.tfevents.1735956966.leonmkim-ROG-Strix-G15CS-G15CS.1975301.0 events.out.tfevents.1735956966.leonmkim-ROG-Strix-G15CS-G15CS.1975301.0 +2025-01-03 21:20:22,482 INFO SenderThread:1975487 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log output.log +2025-01-03 21:20:22,482 INFO SenderThread:1975487 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/diff.patch diff.patch +2025-01-03 21:20:22,487 INFO SenderThread:1975487 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-03 21:20:22,490 INFO SenderThread:1975487 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval/4_eval_4_8ec81cfff6351c2b5db4.mp4 media/videos/eval/4_eval_4_8ec81cfff6351c2b5db4.mp4 +2025-01-03 21:20:22,490 INFO SenderThread:1975487 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval/2_eval_2_a653d18806cfc6db0e07.mp4 media/videos/eval/2_eval_2_a653d18806cfc6db0e07.mp4 +2025-01-03 21:20:22,490 INFO SenderThread:1975487 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval/3_eval_3_a145a501509e034700f1.mp4 media/videos/eval/3_eval_3_a145a501509e034700f1.mp4 +2025-01-03 21:20:22,490 INFO SenderThread:1975487 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval/1_eval_1_91c0aab1f5aa1b260cf6.mp4 media/videos/eval/1_eval_1_91c0aab1f5aa1b260cf6.mp4 +2025-01-03 21:20:22,490 INFO SenderThread:1975487 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval/0_eval_0_29e1a738e40422f19100.mp4 media/videos/eval/0_eval_0_29e1a738e40422f19100.mp4 +2025-01-03 21:20:22,490 INFO SenderThread:1975487 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/table/eval/total_success_rate_ci_6_afac0fa2248ad7b10763.table.json media/table/eval/total_success_rate_ci_6_afac0fa2248ad7b10763.table.json +2025-01-03 21:20:22,491 INFO SenderThread:1975487 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-03 21:20:22,491 INFO SenderThread:1975487 [sender.py:transition_state():614] send defer: 10 +2025-01-03 21:20:22,491 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:20:22,491 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:20:22,491 INFO HandlerThread:1975487 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-03 21:20:22,491 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: defer +2025-01-03 21:20:22,491 INFO SenderThread:1975487 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-03 21:20:22,491 INFO SenderThread:1975487 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 21:20:22,670 INFO wandb-upload_0:1975487 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/requirements.txt +2025-01-03 21:20:22,678 INFO wandb-upload_2:1975487 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/config.yaml +2025-01-03 21:20:22,704 INFO wandb-upload_1:1975487 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/wandb-summary.json +2025-01-03 21:20:22,754 INFO wandb-upload_4:1975487 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:20:22,770 INFO wandb-upload_3:1975487 [upload_job.py:push():130] Uploaded file /tmp/tmpo3rn3d09wandb/h0tghdfr-events.out.tfevents.1735956966.leonmkim-ROG-Strix-G15CS-G15CS.1975301.0 +2025-01-03 21:20:22,971 INFO Thread-11 (_thread_body):1975487 [sender.py:transition_state():614] send defer: 11 +2025-01-03 21:20:22,971 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:20:22,971 INFO HandlerThread:1975487 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-03 21:20:22,971 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: defer +2025-01-03 21:20:22,971 INFO SenderThread:1975487 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-03 21:20:22,971 INFO SenderThread:1975487 [file_pusher.py:join():175] waiting for file pusher +2025-01-03 21:20:22,972 INFO SenderThread:1975487 [sender.py:transition_state():614] send defer: 12 +2025-01-03 21:20:22,972 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:20:22,972 INFO HandlerThread:1975487 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-03 21:20:22,973 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: defer +2025-01-03 21:20:22,973 INFO SenderThread:1975487 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-03 21:20:22,973 INFO SenderThread:1975487 [file_stream.py:finish():601] file stream finish called +2025-01-03 21:20:23,050 INFO SenderThread:1975487 [file_stream.py:finish():605] file stream finish is done +2025-01-03 21:20:23,050 INFO SenderThread:1975487 [sender.py:transition_state():614] send defer: 13 +2025-01-03 21:20:23,050 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:20:23,050 INFO HandlerThread:1975487 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-03 21:20:23,050 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: defer +2025-01-03 21:20:23,050 INFO SenderThread:1975487 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-03 21:20:23,050 INFO SenderThread:1975487 [sender.py:transition_state():614] send defer: 14 +2025-01-03 21:20:23,050 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:20:23,051 INFO HandlerThread:1975487 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-03 21:20:23,051 DEBUG SenderThread:1975487 [sender.py:send():379] send: final +2025-01-03 21:20:23,051 DEBUG SenderThread:1975487 [sender.py:send():379] send: footer +2025-01-03 21:20:23,051 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: defer +2025-01-03 21:20:23,051 INFO SenderThread:1975487 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-03 21:20:23,051 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:20:23,052 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: server_info +2025-01-03 21:20:23,052 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:20:23,052 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:20:23,052 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: get_summary +2025-01-03 21:20:23,052 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: server_info +2025-01-03 21:20:23,053 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-03 21:20:23,055 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:23,115 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:20:23,115 INFO MainThread:1975487 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-03 21:20:23,116 INFO MainThread:1975487 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-03 21:20:23,116 INFO MainThread:1975487 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-03 21:20:23,116 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: shutdown +2025-01-03 21:20:23,116 INFO HandlerThread:1975487 [handler.py:finish():882] shutting down handler +2025-01-03 21:20:24,052 INFO WriterThread:1975487 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/run-nayavtp6.wandb +2025-01-03 21:20:24,115 INFO SenderThread:1975487 [sender.py:finish():1615] shutting down sender +2025-01-03 21:20:24,115 INFO SenderThread:1975487 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 21:20:24,115 INFO SenderThread:1975487 [file_pusher.py:join():175] waiting for file pusher diff --git a/nayavtp6/wandb/debug.log b/nayavtp6/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..bdb88a623e859512d4ab61bbdd1a3c887f37276c --- /dev/null +++ b/nayavtp6/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-03 21:16:06,125 INFO MainThread:1975301 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-03 21:16:06,125 INFO MainThread:1975301 [wandb_setup.py:_flush():76] Configure stats pid to 1975301 +2025-01-03 21:16:06,125 INFO MainThread:1975301 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-03 21:16:06,125 INFO MainThread:1975301 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/settings +2025-01-03 21:16:06,125 INFO MainThread:1975301 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-03 21:16:06,125 INFO MainThread:1975301 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-03 21:16:06,125 INFO MainThread:1975301 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-03 21:16:06,125 INFO MainThread:1975301 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-03 21:16:06,125 INFO MainThread:1975301 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/logs/debug.log +2025-01-03 21:16:06,125 INFO MainThread:1975301 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/logs/debug-internal.log +2025-01-03 21:16:06,125 INFO MainThread:1975301 [wandb_init.py:init():569] calling init triggers +2025-01-03 21:16:06,125 INFO MainThread:1975301 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1926_1', 'wandb_notes': '1926_1_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6'} +2025-01-03 21:16:06,125 INFO MainThread:1975301 [wandb_init.py:init():619] starting backend +2025-01-03 21:16:06,126 INFO MainThread:1975301 [wandb_init.py:init():623] setting up manager +2025-01-03 21:16:06,129 INFO MainThread:1975301 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-03 21:16:06,130 INFO MainThread:1975301 [wandb_init.py:init():631] backend started and connected +2025-01-03 21:16:06,152 INFO MainThread:1975301 [wandb_init.py:init():720] updated telemetry +2025-01-03 21:16:06,158 INFO MainThread:1975301 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-03 21:16:06,412 INFO MainThread:1975301 [wandb_run.py:_on_init():2435] communicating current version +2025-01-03 21:16:06,486 INFO MainThread:1975301 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-03 21:16:06,486 INFO MainThread:1975301 [wandb_init.py:init():804] starting run threads in backend +2025-01-03 21:16:06,831 INFO MainThread:1975301 [wandb_run.py:_console_start():2413] atexit reg +2025-01-03 21:16:06,831 INFO MainThread:1975301 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-03 21:16:06,831 INFO MainThread:1975301 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-03 21:16:06,831 INFO MainThread:1975301 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-03 21:16:06,833 INFO MainThread:1975301 [wandb_init.py:init():847] run started, returning control to user process +2025-01-03 21:16:06,833 INFO MainThread:1975301 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/tb, True +2025-01-03 21:16:11,355 INFO MainThread:1975301 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'} +2025-01-03 21:20:24,117 WARNING MsgRouterThr:1975301 [router.py:message_loop():77] message_loop has been closed diff --git a/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/code/FISH/eval_robot.py b/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/config.yaml b/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3643e6b28d8cf16820228b37e09c9ce72a0b20e1 --- /dev/null +++ b/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 1 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1926_1' +wandb_notes: + desc: null + value: 1926_1_restarted_5 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 1 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6 +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735956966 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: modelsys +left_book_slot: + desc: null + value: twodim diff --git a/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/diff.patch b/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..c97de4502eea91c4c009a6fb51db681afeec762b --- /dev/null +++ b/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/diff.patch @@ -0,0 +1,57 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 57a851f..9234106 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -138,12 +138,19 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' ++# wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1922_2' # seed 2 ++# wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1006_0' ++wandb_run_id: '1926_1' # seed 1 ++# wandb_run_id: '1930_0' # dataset shuffle seed 1 ++ ++ + + # RGBD+mask+contact(contact+DTC)+act history +-wandb_run_id: '1745_0' ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 71a5d1c..c7eadce 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -159,8 +159,10 @@ run_id_list = [ + ] + + run_id_list = [ +- # '1006_0', +- '1745_0' ++ # '1922_1', ++ # '1922_2', ++ '1926_1', ++ '1930_0', + ] + + checkpoint_epoch = 12000 +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index 6f97e26..285afc8 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,8 +330,8 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.005 +- self.move_to_rot_tolerance = 1.0 ++ self.move_to_trans_tolerance = 0.01 ++ self.move_to_rot_tolerance = 2.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) + self.move_to_target_timeout = 5.0 diff --git a/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json b/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json new file mode 100644 index 0000000000000000000000000000000000000000..faa9910a25d3f0826c10a7b81de69b8b94278cd1 --- /dev/null +++ b/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.0, 0, 0.5363743901179997, 0, 0.48415964842159465, 0, 0.21277225023292168, 0, 0.1673506349990479]]} \ No newline at end of file diff --git a/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/table/eval/total_success_rate_ci_6_afac0fa2248ad7b10763.table.json b/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/table/eval/total_success_rate_ci_6_afac0fa2248ad7b10763.table.json new file mode 100644 index 0000000000000000000000000000000000000000..6a500b95d1b4043ee298ae867c61accbbdb4fdfa --- /dev/null +++ b/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/table/eval/total_success_rate_ci_6_afac0fa2248ad7b10763.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[9, 15, 0.6, 0.339076846870182, 0.8388251363553458, 0.3342101574275672, 0.8033371120084625, 0.3638527542624972, 0.7916983216417566, 0.4603962749321989, 0.7620583213341641]]} \ No newline at end of file diff --git a/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval/0_eval_0_29e1a738e40422f19100.mp4 b/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval/0_eval_0_29e1a738e40422f19100.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..438ef780f399c137026ac497af9eda8a3d3292e7 --- /dev/null +++ b/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval/0_eval_0_29e1a738e40422f19100.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29e1a738e40422f191005ecb6090c8f1a958b08c29ad80f4472e2a4a928eded4 +size 565195 diff --git a/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval/1_eval_1_91c0aab1f5aa1b260cf6.mp4 b/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval/1_eval_1_91c0aab1f5aa1b260cf6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7ae1d8c194a1bc40505eb24ab6344f3138f1ea99 --- /dev/null +++ b/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval/1_eval_1_91c0aab1f5aa1b260cf6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91c0aab1f5aa1b260cf6cc217e49c394fc6b03957edac0e7c36256b102aa7edb +size 561687 diff --git a/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval/2_eval_2_a653d18806cfc6db0e07.mp4 b/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval/2_eval_2_a653d18806cfc6db0e07.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b8e220ecbb4960726d7193c984be04a4e5b164de --- /dev/null +++ b/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval/2_eval_2_a653d18806cfc6db0e07.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a653d18806cfc6db0e07b96db57207d1ae0bb118ab46768e8215693a45563227 +size 581960 diff --git a/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval/3_eval_3_a145a501509e034700f1.mp4 b/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval/3_eval_3_a145a501509e034700f1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0cb40052c3bfc42b0b0d5dfb98cf4002976d9cb4 --- /dev/null +++ b/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval/3_eval_3_a145a501509e034700f1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a145a501509e034700f1d2965bd668794ffea38d3be9b9defb32ea120f938937 +size 966517 diff --git a/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval/4_eval_4_8ec81cfff6351c2b5db4.mp4 b/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval/4_eval_4_8ec81cfff6351c2b5db4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b24ec897e8c2e93c55d183567ff7ba0a58e12c67 --- /dev/null +++ b/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval/4_eval_4_8ec81cfff6351c2b5db4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ec81cfff6351c2b5db45e5552d306f19b1d1dd60a84948557b9ee24dc8bec45 +size 789769 diff --git a/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log b/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..bfaff708e1f51d898147ce2d9ecc765194598ae3 --- /dev/null +++ b/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log @@ -0,0 +1,292 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735956971.356302]: resetting environment +[INFO] [1735956971.360822]: cleared current plan +[INFO] [1735956971.361607]: moving to home +[INFO] [1735956972.364111]: reached home +[INFO] [1735956972.365452]: reset action history +[INFO] [1735956975.075572]: environment reset +Starting episode 0 +[INFO] [1735956975.079703]: resetting environment +[INFO] [1735956975.084105]: cleared current plan +[INFO] [1735956975.085377]: moving to home +[INFO] [1735956976.087470]: reached home +[INFO] [1735956976.088972]: reset action history +[INFO] [1735956978.798784]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1735956987.947610]: resetting environment +[INFO] [1735956987.952272]: cleared current plan +[INFO] [1735956987.954800]: moving to home +[INFO] [1735956988.958341]: reached home +[INFO] [1735956988.959578]: reset action history +[INFO] [1735956991.666730]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/episode_rosbags/episode_0_2025-01-03-21-16-31.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1975301.log +started roslaunch server http://158.130.50.37:46437/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1975591] +process[print_text-2]: started with pid [1975615] +process[rosbag_record-3]: started with pid [1975616] +started bagging! +For topic gripper_width: timestamp difference is 101278018 for nearest: 1735957003128241729 - target: 1735957003026963711 at idx 160 +preempting episode +[INFO] [1735957003.427234]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1735957004.167587]: Stored episode 1. +[INFO] [1735957004.167834]: Saving video... +[INFO] [1735957004.412593]: Video saved! +Starting episode 1 +[INFO] [1735957004.423884]: resetting environment +[INFO] [1735957004.431545]: cleared current plan +[INFO] [1735957004.441779]: moving to home +[INFO] [1735957007.945262]: reached home +[INFO] [1735957007.945833]: reset action history +[INFO] [1735957010.655120]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735957029.327340]: resetting environment +[INFO] [1735957029.333317]: cleared current plan +[INFO] [1735957029.333717]: moving to home +[INFO] [1735957030.335280]: reached home +[INFO] [1735957030.335686]: reset action history +[INFO] [1735957033.043708]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/episode_rosbags/episode_1_2025-01-03-21-17-13.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1975301.log +started roslaunch server http://158.130.50.37:37135/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1975749] +process[print_text-5]: started with pid [1975750] +process[rosbag_record-6]: started with pid [1975774] +started bagging! +preempting episode +[INFO] [1735957044.821522]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1735957045.635835]: Stored episode 2. +[INFO] [1735957045.636061]: Saving video... +[INFO] [1735957045.918200]: Video saved! +Starting episode 2 +[INFO] [1735957045.965294]: resetting environment +[INFO] [1735957045.989485]: cleared current plan +[INFO] [1735957045.993797]: moving to home +[INFO] [1735957049.612041]: reached home +[INFO] [1735957049.614507]: reset action history +[INFO] [1735957052.398610]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735957071.888423]: resetting environment +[INFO] [1735957071.895612]: cleared current plan +[INFO] [1735957071.895809]: moving to home +[INFO] [1735957072.897271]: reached home +[INFO] [1735957072.897718]: reset action history +[INFO] [1735957075.606747]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/episode_rosbags/episode_2_2025-01-03-21-17-55.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1975301.log +started roslaunch server http://158.130.50.37:42331/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1975904] +process[print_text-8]: started with pid [1975905] +process[rosbag_record-9]: started with pid [1975929] +started bagging! +preempting episode +[INFO] [1735957087.452813]: Storing episode... +[rosbag_record-9] killing on exit +[pub_text-7] killing on exit +[print_text-8] killing on exit +[INFO] [1735957088.216200]: Stored episode 3. +[INFO] [1735957088.220573]: Saving video... +[WARN] [1735957088.586201]: Plan exhausted +[WARN] [1735957088.630589]: Plan exhausted +[WARN] [1735957088.666771]: Plan exhausted +[WARN] [1735957088.706752]: Plan exhausted +[INFO] [1735957088.659896]: Video saved! +Starting episode 3 +[INFO] [1735957088.713506]: resetting environment +[INFO] [1735957088.732953]: cleared current plan +[INFO] [1735957088.733603]: moving to home +[INFO] [1735957092.137578]: reached home +[INFO] [1735957092.140645]: reset action history +[INFO] [1735957094.981215]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735957108.752247]: resetting environment +[INFO] [1735957108.760482]: cleared current plan +[INFO] [1735957108.760709]: moving to home +[INFO] [1735957109.768457]: reached home +[INFO] [1735957109.770721]: reset action history +[INFO] [1735957112.675490]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/episode_rosbags/episode_3_2025-01-03-21-18-32.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1975301.log +started roslaunch server http://158.130.50.37:43039/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1976068] +process[print_text-11]: started with pid [1976069] +process[rosbag_record-12]: started with pid [1976070] +started bagging! +For topic gripper_width: timestamp difference is -106152961 for nearest: 1735957117561574986 - target: 1735957117667727947 at idx 66 +For topic gripper_width: timestamp difference is -111862675 for nearest: 1735957122261610731 - target: 1735957122373473406 at idx 122 +preempting episode +[INFO] [1735957134.442548]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[WARN] [1735957135.683529]: Plan exhausted +[WARN] [1735957135.724045]: Plan exhausted +[WARN] [1735957135.763525]: Plan exhausted +[WARN] [1735957135.804086]: Plan exhausted +[WARN] [1735957135.844586]: Plan exhausted +[WARN] [1735957135.886323]: Plan exhausted +[WARN] [1735957135.923190]: Plan exhausted +[WARN] [1735957135.973470]: Plan exhausted +[INFO] [1735957135.311875]: Stored episode 4. +[INFO] [1735957135.317478]: Saving video... +[INFO] [1735957135.899896]: Video saved! +Starting episode 4 +[INFO] [1735957135.962934]: resetting environment +[INFO] [1735957135.985021]: cleared current plan +[INFO] [1735957135.985317]: moving to home +[INFO] [1735957139.589092]: reached home +[INFO] [1735957139.598547]: reset action history +[INFO] [1735957142.403270]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735957155.715886]: resetting environment +[INFO] [1735957155.755075]: cleared current plan +[INFO] [1735957155.760907]: moving to home +[INFO] [1735957156.768560]: reached home +[INFO] [1735957156.775216]: reset action history +[INFO] [1735957159.572171]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/episode_rosbags/episode_4_2025-01-03-21-19-19.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1975301.log +started roslaunch server http://158.130.50.37:46135/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1976186] +process[print_text-14]: started with pid [1976187] +process[rosbag_record-15]: started with pid [1976211] +started bagging! +For topic gripper_width: timestamp difference is 105358455 for nearest: 1735957173328239295 - target: 1735957173222880840 at idx 187 +For topic gripper_width: timestamp difference is 106385476 for nearest: 1735957174628240353 - target: 1735957174521854877 at idx 203 +For topic EE_pose: timestamp difference is 110025419 for nearest: 1735957175500336183 - target: 1735957175390310764 at idx 0 +For topic EE_pose: timestamp difference is 254988975 for nearest: 1735957175645299739 - target: 1735957175390310764 at idx 0 +For topic EE_pose: timestamp difference is 374936698 for nearest: 1735957175765247462 - target: 1735957175390310764 at idx 0 +[WARN] [1735957178.166690]: Plan exhausted +preempting episode +[INFO] [1735957176.939369]: Storing episode... +[print_text-14] killing on exit +[pub_text-13] killing on exit +[rosbag_record-15] killing on exit +[INFO] [1735957177.708697]: Stored episode 5. +[INFO] [1735957177.713953]: Saving video... +[INFO] [1735957178.216078]: Video saved! +[INFO] [1735957178.262917]: resetting environment +[INFO] [1735957178.334066]: cleared current plan +[INFO] [1735957178.336622]: moving to home +[WARN] [1735957178.203195]: Plan exhausted +[WARN] [1735957178.244087]: Plan exhausted +[WARN] [1735957178.283187]: Plan exhausted +[WARN] [1735957178.327640]: Plan exhausted +[INFO] [1735957182.039251]: reached home +[INFO] [1735957182.045354]: reset action history +[INFO] [1735957184.891640]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/requirements.txt b/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/wandb-metadata.json b/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..231074584bfc2e0bb162422ec376fe733db07eac --- /dev/null +++ b/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-04T02:16:06.510029", + "startedAt": "2025-01-04T02:16:06.123552", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "c537eb6734155592809c2a7db5cca971ec186d72" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3749.2645, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.999, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.985, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 807.209831237793 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/wandb-summary.json b/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..fc578a1300af65b744254e73030d80b5992f8aab --- /dev/null +++ b/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "29e1a738e40422f191005ecb6090c8f1a958b08c29ad80f4472e2a4a928eded4", "size": 565195, "path": "media/videos/eval/0_eval_0_29e1a738e40422f19100.mp4"}, "global_step": 281, "_timestamp": 1735957205.4980323, "_runtime": 239.36700224876404, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "91c0aab1f5aa1b260cf6cc217e49c394fc6b03957edac0e7c36256b102aa7edb", "size": 561687, "path": "media/videos/eval/1_eval_1_91c0aab1f5aa1b260cf6.mp4"}, "eval/num_success": 0.0, "episode": 15.0, "eval/success_rate": 0.0, "eval/success_rate_lb_0.01": 0.0, "eval/success_rate_ub_0.01": 0.5363743901252747, "eval/success_rate_lb_0.025": 0.0, "eval/success_rate_ub_0.025": 0.4841596484184265, "eval/success_rate_lb_0.05": 0.0, "eval/success_rate_ub_0.05": 0.21277225017547607, "eval/success_rate_lb_0.1": 0.0, "eval/success_rate_ub_0.1": 0.167350634932518, "eval/2_eval": {"_type": "video-file", "sha256": "a653d18806cfc6db0e07b96db57207d1ae0bb118ab46768e8215693a45563227", "size": 581960, "path": "media/videos/eval/2_eval_2_a653d18806cfc6db0e07.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "a145a501509e034700f1d2965bd668794ffea38d3be9b9defb32ea120f938937", "size": 966517, "path": "media/videos/eval/3_eval_3_a145a501509e034700f1.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "8ec81cfff6351c2b5db45e5552d306f19b1d1dd60a84948557b9ee24dc8bec45", "size": 789769, "path": "media/videos/eval/4_eval_4_8ec81cfff6351c2b5db4.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "e6508f0181aa84573747f0d380a2c6725005451ce9eb7fcdf77e1f39c90f7528", "size": 332, "artifact_path": "wandb-client-artifact://57d63r1upbiw8vz8nifxl6hzwzs87qczpiezl1w0228yg50ur2tcvmrib1kdiqfbexon0xz2clf478uao21eh3m1wkftt5lmrg3pu7h7chcyn9e3vhcyy23cikrba6an/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://xdhqx46gduty1dh6zfcz73vkv0yynz9loctj165zdwc8b5gbzr8y6sdm5hexs45gdbf8z0y9dnr7xjf4ltzvbh7skxp3s2ugh3ejv5lvg2r7h8wronjobvlx7dkhbtay:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 9, "total_num_episodes": 15, "total_success_rate": 0.6, "total_success_rate_lb_0.01": 0.339076846870182, "total_success_rate_ub_0.01": 0.8388251363553458, "total_success_rate_lb_0.025": 0.3342101574275672, "total_success_rate_ub_0.025": 0.8033371120084625, "total_success_rate_lb_0.05": 0.3638527542624972, "total_success_rate_ub_0.05": 0.7916983216417566, "total_success_rate_lb_0.1": 0.4603962749321989, "total_success_rate_ub_0.1": 0.7620583213341641, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "afac0fa2248ad7b10763717db7ca66e988e833faf4f5501ba55531e68cfdc4ee", "size": 503, "artifact_path": "wandb-client-artifact://9kpfiqa8cf8zmjg69s2sjt9ccr8v9eo6dmorfluaztjovuwe2d7q9qj8etcdxpsgliouponaqif2gk92te7m8vh5gm63bli1z7un9vgnxsu2g1qfjbpbm79occdlpdzk/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://0vi9n01jmkvulhe35f8lkevjx4tiglm4jpl9n2ph2btw8k6decjfxom03fm2ij0wq0uextmalwih6b9ihpdsktp4rsrbpnmqhyrx8q204dfrkpqs4himcdy5d6z8ul73:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_afac0fa2248ad7b10763.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 243}, "eval/total_success_rate": 0.6000000238418579, "eval/total_success_rate_lb_0.01": 0.33907684683799744, "eval/total_success_rate_ub_0.01": 0.8388251066207886, "eval/total_success_rate_lb_0.025": 0.3342101573944092, "eval/total_success_rate_ub_0.025": 0.8033370971679688, "eval/total_success_rate_lb_0.05": 0.36385276913642883, "eval/total_success_rate_ub_0.05": 0.7916983366012573, "eval/total_success_rate_lb_0.1": 0.46039628982543945, "eval/total_success_rate_ub_0.1": 0.7620583176612854} \ No newline at end of file diff --git a/nayavtp6/wandb/run-20250103_211606-nayavtp6/logs/debug-internal.log b/nayavtp6/wandb/run-20250103_211606-nayavtp6/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..7ff7ccd4d26f8860de957ab8104917580becb96d --- /dev/null +++ b/nayavtp6/wandb/run-20250103_211606-nayavtp6/logs/debug-internal.log @@ -0,0 +1,738 @@ +2025-01-03 21:16:06,131 INFO StreamThr :1975487 [internal.py:wandb_internal():85] W&B internal server running at pid: 1975487, started at: 2025-01-03 21:16:06.130701 +2025-01-03 21:16:06,132 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status +2025-01-03 21:16:06,152 INFO WriterThread:1975487 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/run-nayavtp6.wandb +2025-01-03 21:16:06,152 DEBUG SenderThread:1975487 [sender.py:send():379] send: header +2025-01-03 21:16:06,160 DEBUG SenderThread:1975487 [sender.py:send():379] send: run +2025-01-03 21:16:06,394 INFO SenderThread:1975487 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files +2025-01-03 21:16:06,395 INFO SenderThread:1975487 [sender.py:_start_run_threads():1188] run started: nayavtp6 with start time 1735956966.13103 +2025-01-03 21:16:06,413 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: check_version +2025-01-03 21:16:06,413 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: check_version +2025-01-03 21:16:06,492 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: run_start +2025-01-03 21:16:06,496 DEBUG HandlerThread:1975487 [system_info.py:__init__():26] System info init +2025-01-03 21:16:06,497 DEBUG HandlerThread:1975487 [system_info.py:__init__():41] System info init done +2025-01-03 21:16:06,497 INFO HandlerThread:1975487 [system_monitor.py:start():194] Starting system monitor +2025-01-03 21:16:06,497 INFO SystemMonitor:1975487 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-03 21:16:06,497 INFO HandlerThread:1975487 [system_monitor.py:probe():214] Collecting system info +2025-01-03 21:16:06,497 INFO SystemMonitor:1975487 [interfaces.py:start():188] Started cpu monitoring +2025-01-03 21:16:06,498 INFO SystemMonitor:1975487 [interfaces.py:start():188] Started disk monitoring +2025-01-03 21:16:06,498 INFO SystemMonitor:1975487 [interfaces.py:start():188] Started gpu monitoring +2025-01-03 21:16:06,499 INFO SystemMonitor:1975487 [interfaces.py:start():188] Started memory monitoring +2025-01-03 21:16:06,499 INFO SystemMonitor:1975487 [interfaces.py:start():188] Started network monitoring +2025-01-03 21:16:06,509 DEBUG HandlerThread:1975487 [system_info.py:probe():152] Probing system +2025-01-03 21:16:06,511 DEBUG HandlerThread:1975487 [system_info.py:_probe_git():137] Probing git +2025-01-03 21:16:06,515 DEBUG HandlerThread:1975487 [system_info.py:_probe_git():145] Probing git done +2025-01-03 21:16:06,516 DEBUG HandlerThread:1975487 [system_info.py:probe():200] Probing system done +2025-01-03 21:16:06,516 DEBUG HandlerThread:1975487 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-04T02:16:06.510029', 'startedAt': '2025-01-04T02:16:06.123552', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'c537eb6734155592809c2a7db5cca971ec186d72'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.2645, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.999, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.985, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 807.209831237793}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-03 21:16:06,516 INFO HandlerThread:1975487 [system_monitor.py:probe():224] Finished collecting system info +2025-01-03 21:16:06,516 INFO HandlerThread:1975487 [system_monitor.py:probe():227] Publishing system info +2025-01-03 21:16:06,516 DEBUG HandlerThread:1975487 [system_info.py:_save_code():44] Saving code +2025-01-03 21:16:06,517 DEBUG HandlerThread:1975487 [system_info.py:_save_code():67] Saving code done +2025-01-03 21:16:06,517 DEBUG HandlerThread:1975487 [system_info.py:_save_patches():84] Saving git patches +2025-01-03 21:16:06,579 DEBUG HandlerThread:1975487 [system_info.py:_save_patches():126] Saving git patches done +2025-01-03 21:16:06,579 INFO HandlerThread:1975487 [system_monitor.py:probe():229] Finished publishing system info +2025-01-03 21:16:06,583 DEBUG SenderThread:1975487 [sender.py:send():379] send: files +2025-01-03 21:16:06,583 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-03 21:16:06,583 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-03 21:16:06,583 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-03 21:16:06,831 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: python_packages +2025-01-03 21:16:06,831 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: python_packages +2025-01-03 21:16:06,831 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:16:06,832 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:06,832 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:16:06,833 INFO HandlerThread:1975487 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/tb" + save: true +} +control { + relay_id: "729b5f9de7ed46d2a8965422843bb4ce" +} +_info { + stream_id: "nayavtp6" +} + +2025-01-03 21:16:06,833 DEBUG HandlerThread:1975487 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-03 21:16:06,845 INFO wandb-upload_0:1975487 [upload_job.py:push():130] Uploaded file /tmp/tmpo3rn3d09wandb/oxpgz1fy-wandb-metadata.json +2025-01-03 21:16:06,854 INFO wandb-upload_1:1975487 [upload_job.py:push():130] Uploaded file /tmp/tmpo3rn3d09wandb/2wylazit-code/FISH/eval_robot.py +2025-01-03 21:16:06,877 INFO wandb-upload_2:1975487 [upload_job.py:push():130] Uploaded file /tmp/tmpo3rn3d09wandb/jcr2f0l8-diff.patch +2025-01-03 21:16:06,938 DEBUG SenderThread:1975487 [sender.py:send():379] send: telemetry +2025-01-03 21:16:06,938 DEBUG SenderThread:1975487 [sender.py:send():379] send: tbrecord +2025-01-03 21:16:06,939 DEBUG SenderThread:1975487 [sender.py:send():379] send: files +2025-01-03 21:16:06,939 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file events.out.tfevents.1735956966.leonmkim-ROG-Strix-G15CS-G15CS.1975301.0 with policy live +2025-01-03 21:16:07,192 INFO wandb-upload_0:1975487 [upload_job.py:push():130] Uploaded file /tmp/tmpo3rn3d09wandb/ehlc0r0c-events.out.tfevents.1735956966.leonmkim-ROG-Strix-G15CS-G15CS.1975301.0 +2025-01-03 21:16:07,396 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/wandb-metadata.json +2025-01-03 21:16:07,396 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/requirements.txt +2025-01-03 21:16:07,396 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/code/FISH/eval_robot.py +2025-01-03 21:16:07,397 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/events.out.tfevents.1735956966.leonmkim-ROG-Strix-G15CS-G15CS.1975301.0 +2025-01-03 21:16:07,397 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/diff.patch +2025-01-03 21:16:07,397 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:16:07,397 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/code/FISH +2025-01-03 21:16:07,397 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/code +2025-01-03 21:16:07,831 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:08,831 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:09,395 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:16:09,832 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:10,832 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:11,356 DEBUG SenderThread:1975487 [sender.py:send():379] send: config +2025-01-03 21:16:11,356 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:16:11,832 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:12,832 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:13,397 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:16:13,832 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:14,832 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:15,833 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:16,832 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:16:16,833 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:17,398 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:16:17,833 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:18,841 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:19,834 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:20,834 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:21,399 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:16:21,831 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:16:21,831 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:16:21,873 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:21,952 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:16:22,834 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:23,834 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:24,834 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:25,835 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:26,835 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:27,833 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:16:27,835 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:28,835 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:29,401 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:16:29,835 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:30,835 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:31,402 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:16:31,835 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:32,835 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:32,971 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:16:33,402 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:16:33,836 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:34,836 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:35,836 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:36,831 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:16:36,831 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:16:36,873 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:37,836 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:38,836 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:38,936 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:16:39,404 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/config.yaml +2025-01-03 21:16:39,836 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:40,836 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:41,836 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:42,836 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:43,837 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:44,168 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:16:44,422 DEBUG SenderThread:1975487 [sender.py:send():379] send: files +2025-01-03 21:16:44,422 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_29e1a738e40422f19100.mp4 with policy now +2025-01-03 21:16:44,423 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:16:44,425 DEBUG SenderThread:1975487 [sender.py:send():379] send: history +2025-01-03 21:16:44,425 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:16:44,426 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:16:44,696 INFO wandb-upload_2:1975487 [upload_job.py:push():130] Uploaded file /tmp/tmpo3rn3d09wandb/euxkfv4q-media/videos/eval/0_eval_0_29e1a738e40422f19100.mp4 +2025-01-03 21:16:44,837 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:45,406 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:16:45,406 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval/0_eval_0_29e1a738e40422f19100.mp4 +2025-01-03 21:16:45,406 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/wandb-summary.json +2025-01-03 21:16:45,406 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval +2025-01-03 21:16:45,406 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos +2025-01-03 21:16:45,406 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media +2025-01-03 21:16:45,684 INFO wandb-upload_0:1975487 [upload_job.py:push():130] Uploaded file /tmp/tmpo3rn3d09wandb/wxyv8isk-media/videos/eval/0_eval_0_29e1a738e40422f19100.mp4 +2025-01-03 21:16:45,837 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:46,837 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:47,838 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:48,837 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:49,407 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:16:49,838 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:49,989 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:16:50,838 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:51,831 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:16:51,831 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:16:51,873 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:52,838 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:53,408 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:16:53,838 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:54,838 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:55,838 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:55,939 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:16:56,838 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:57,839 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:58,839 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:16:59,839 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:00,839 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:01,833 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:17:01,839 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:02,839 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:03,839 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:04,839 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:05,839 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:06,499 DEBUG SystemMonitor:1975487 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-03 21:17:06,500 DEBUG SenderThread:1975487 [sender.py:send():379] send: stats +2025-01-03 21:17:06,831 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:17:06,832 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:17:06,873 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:06,937 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:17:07,840 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:08,840 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:09,414 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/events.out.tfevents.1735956966.leonmkim-ROG-Strix-G15CS-G15CS.1975301.0 +2025-01-03 21:17:09,661 INFO wandb-upload_2:1975487 [upload_job.py:push():130] Uploaded file /tmp/tmpo3rn3d09wandb/5yur69kf-events.out.tfevents.1735956966.leonmkim-ROG-Strix-G15CS-G15CS.1975301.0 +2025-01-03 21:17:09,840 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:10,840 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:11,414 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:17:11,840 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:12,834 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:17:12,840 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:13,840 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:14,840 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:15,416 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:17:15,841 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:16,841 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:17,834 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:17:17,841 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:18,841 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:19,841 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:20,841 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:21,832 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:17:21,832 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:17:21,873 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:22,841 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:22,891 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:17:23,842 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:24,853 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:25,853 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:25,965 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:17:25,965 DEBUG SenderThread:1975487 [sender.py:send():379] send: files +2025-01-03 21:17:25,965 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_91c0aab1f5aa1b260cf6.mp4 with policy now +2025-01-03 21:17:25,975 DEBUG SenderThread:1975487 [sender.py:send():379] send: history +2025-01-03 21:17:25,975 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:17:25,975 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:17:26,238 INFO wandb-upload_0:1975487 [upload_job.py:push():130] Uploaded file /tmp/tmpo3rn3d09wandb/pfplswdq-media/videos/eval/1_eval_1_91c0aab1f5aa1b260cf6.mp4 +2025-01-03 21:17:26,420 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/wandb-summary.json +2025-01-03 21:17:26,420 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval/1_eval_1_91c0aab1f5aa1b260cf6.mp4 +2025-01-03 21:17:26,420 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval +2025-01-03 21:17:26,678 INFO wandb-upload_2:1975487 [upload_job.py:push():130] Uploaded file /tmp/tmpo3rn3d09wandb/d6kwnv5m-media/videos/eval/1_eval_1_91c0aab1f5aa1b260cf6.mp4 +2025-01-03 21:17:26,847 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:27,420 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:17:27,847 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:28,844 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:17:28,850 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:29,848 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:30,849 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:31,421 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:17:31,850 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:32,846 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:33,422 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:17:33,846 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:34,441 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:17:34,848 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:35,846 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:36,500 DEBUG SenderThread:1975487 [sender.py:send():379] send: stats +2025-01-03 21:17:36,832 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:17:36,832 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:17:36,873 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:37,847 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:38,847 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:39,847 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:39,982 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:17:40,847 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:41,850 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:42,849 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:43,848 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:44,848 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:45,836 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:17:45,848 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:46,854 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:47,849 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:48,849 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:49,851 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:50,836 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:17:50,851 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:51,833 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:17:51,833 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:17:51,873 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:52,232 DEBUG SenderThread:1975487 [sender.py:send():379] send: history +2025-01-03 21:17:52,232 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:17:52,234 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:17:52,428 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/wandb-summary.json +2025-01-03 21:17:52,428 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/events.out.tfevents.1735956966.leonmkim-ROG-Strix-G15CS-G15CS.1975301.0 +2025-01-03 21:17:52,853 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:53,429 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:17:53,852 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:54,852 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:55,429 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:17:55,857 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:55,863 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:17:56,855 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:57,430 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:17:57,855 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:58,855 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:17:59,855 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:00,855 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:01,836 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:18:01,856 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:02,856 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:03,856 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:04,856 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:05,856 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:06,501 DEBUG SenderThread:1975487 [sender.py:send():379] send: stats +2025-01-03 21:18:06,832 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:18:06,832 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:18:06,877 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:06,911 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:18:07,857 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:08,704 DEBUG SenderThread:1975487 [sender.py:send():379] send: files +2025-01-03 21:18:08,704 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_a653d18806cfc6db0e07.mp4 with policy now +2025-01-03 21:18:08,709 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:18:08,709 DEBUG SenderThread:1975487 [sender.py:send():379] send: history +2025-01-03 21:18:08,709 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:18:08,710 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:18:08,857 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:08,950 INFO wandb-upload_0:1975487 [upload_job.py:push():130] Uploaded file /tmp/tmpo3rn3d09wandb/dyha6n7f-media/videos/eval/2_eval_2_a653d18806cfc6db0e07.mp4 +2025-01-03 21:18:09,434 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/wandb-summary.json +2025-01-03 21:18:09,434 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:18:09,434 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval/2_eval_2_a653d18806cfc6db0e07.mp4 +2025-01-03 21:18:09,434 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval +2025-01-03 21:18:09,698 INFO wandb-upload_2:1975487 [upload_job.py:push():130] Uploaded file /tmp/tmpo3rn3d09wandb/9rq9zg5h-media/videos/eval/2_eval_2_a653d18806cfc6db0e07.mp4 +2025-01-03 21:18:09,869 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:10,872 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:11,435 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:18:11,869 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:12,139 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:18:12,872 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:13,435 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:18:13,869 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:14,870 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:15,872 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:16,874 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:17,437 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:18:17,850 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:18:17,878 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:18,873 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:19,874 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:20,877 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:21,835 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:18:21,836 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:18:21,877 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:22,877 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:22,969 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:18:23,875 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:24,878 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:25,875 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:26,882 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:27,879 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:28,730 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:18:28,877 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:29,057 DEBUG SenderThread:1975487 [sender.py:send():379] send: history +2025-01-03 21:18:29,057 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:18:29,058 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:18:29,441 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/wandb-summary.json +2025-01-03 21:18:29,441 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/events.out.tfevents.1735956966.leonmkim-ROG-Strix-G15CS-G15CS.1975301.0 +2025-01-03 21:18:29,676 INFO wandb-upload_0:1975487 [upload_job.py:push():130] Uploaded file /tmp/tmpo3rn3d09wandb/gutyp7n3-events.out.tfevents.1735956966.leonmkim-ROG-Strix-G15CS-G15CS.1975301.0 +2025-01-03 21:18:29,883 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:30,888 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:31,441 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:18:31,877 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:32,877 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:33,878 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:34,111 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:18:34,878 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:35,443 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:18:35,878 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:36,501 DEBUG SenderThread:1975487 [sender.py:send():379] send: stats +2025-01-03 21:18:36,833 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:18:36,833 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:18:36,878 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:37,878 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:38,878 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:39,444 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:18:39,801 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:18:39,878 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:40,878 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:41,880 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:42,880 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:43,446 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:18:43,880 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:44,881 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:45,489 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:18:45,881 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:46,881 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:47,883 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:48,882 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:49,884 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:50,847 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:18:50,884 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:51,833 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:18:51,833 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:18:51,884 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:52,884 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:53,884 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:54,886 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:55,450 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:18:55,887 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:18:55,889 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:55,953 DEBUG SenderThread:1975487 [sender.py:send():379] send: files +2025-01-03 21:18:55,953 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_a145a501509e034700f1.mp4 with policy now +2025-01-03 21:18:55,957 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:18:55,957 DEBUG SenderThread:1975487 [sender.py:send():379] send: history +2025-01-03 21:18:55,957 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:18:55,958 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:18:56,210 INFO wandb-upload_2:1975487 [upload_job.py:push():130] Uploaded file /tmp/tmpo3rn3d09wandb/1jybtjn8-media/videos/eval/3_eval_3_a145a501509e034700f1.mp4 +2025-01-03 21:18:56,450 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/wandb-summary.json +2025-01-03 21:18:56,450 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval/3_eval_3_a145a501509e034700f1.mp4 +2025-01-03 21:18:56,450 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval +2025-01-03 21:18:56,708 INFO wandb-upload_0:1975487 [upload_job.py:push():130] Uploaded file /tmp/tmpo3rn3d09wandb/amwdcsax-media/videos/eval/3_eval_3_a145a501509e034700f1.mp4 +2025-01-03 21:18:56,889 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:57,450 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:18:57,892 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:58,902 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:18:59,898 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:00,905 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:01,452 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:19:01,637 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:19:01,904 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:02,899 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:03,452 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:19:03,901 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:04,901 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:05,904 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:06,502 DEBUG SenderThread:1975487 [sender.py:send():379] send: stats +2025-01-03 21:19:06,835 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:19:06,835 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:19:06,915 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:06,991 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:19:07,916 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:08,908 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:09,910 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:10,903 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:11,916 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:12,859 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:19:12,914 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:13,912 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:14,913 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:15,912 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:16,112 DEBUG SenderThread:1975487 [sender.py:send():379] send: history +2025-01-03 21:19:16,112 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:19:16,113 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:19:16,457 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/wandb-summary.json +2025-01-03 21:19:16,457 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/events.out.tfevents.1735956966.leonmkim-ROG-Strix-G15CS-G15CS.1975301.0 +2025-01-03 21:19:16,920 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:17,457 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:19:17,913 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:18,858 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:19:18,917 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:19,458 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:19:19,911 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:20,911 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:21,459 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:19:21,833 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:19:21,834 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:19:21,911 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:22,911 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:23,893 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:19:23,912 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:24,912 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:25,912 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:26,912 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:27,912 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:28,912 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:29,853 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:19:29,913 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:30,912 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:31,913 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:32,913 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:33,913 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:34,915 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:35,463 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:19:35,677 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:19:35,921 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:36,502 DEBUG SenderThread:1975487 [sender.py:send():379] send: stats +2025-01-03 21:19:36,833 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:19:36,834 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:19:36,913 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:37,464 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:19:37,918 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:38,256 DEBUG SenderThread:1975487 [sender.py:send():379] send: files +2025-01-03 21:19:38,256 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_8ec81cfff6351c2b5db4.mp4 with policy now +2025-01-03 21:19:38,260 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:19:38,261 DEBUG SenderThread:1975487 [sender.py:send():379] send: history +2025-01-03 21:19:38,261 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:19:38,261 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:19:38,465 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/wandb-summary.json +2025-01-03 21:19:38,465 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval/4_eval_4_8ec81cfff6351c2b5db4.mp4 +2025-01-03 21:19:38,465 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval +2025-01-03 21:19:38,500 INFO wandb-upload_2:1975487 [upload_job.py:push():130] Uploaded file /tmp/tmpo3rn3d09wandb/g9puoe08-media/videos/eval/4_eval_4_8ec81cfff6351c2b5db4.mp4 +2025-01-03 21:19:38,752 INFO wandb-upload_0:1975487 [upload_job.py:push():130] Uploaded file /tmp/tmpo3rn3d09wandb/e8pp42sj-media/videos/eval/4_eval_4_8ec81cfff6351c2b5db4.mp4 +2025-01-03 21:19:38,920 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:39,465 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:19:39,921 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:40,933 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:41,337 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:19:41,466 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:19:41,928 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:42,939 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:43,466 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:19:43,934 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:44,941 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:45,940 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:46,865 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:19:46,958 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:47,468 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:19:47,955 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:48,947 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:49,951 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:50,952 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:51,835 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:19:51,836 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:19:51,902 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:19:51,956 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:52,955 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:53,958 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:54,959 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:55,959 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:56,959 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:57,861 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:19:57,961 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:58,958 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:19:59,966 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:00,961 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:01,959 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:02,880 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:20:02,979 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:03,980 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:04,965 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:05,029 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 21:20:05,030 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 21:20:05,189 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:20:05,474 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/events.out.tfevents.1735956966.leonmkim-ROG-Strix-G15CS-G15CS.1975301.0 +2025-01-03 21:20:05,474 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:20:05,474 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-03 21:20:05,474 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media +2025-01-03 21:20:05,474 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/table +2025-01-03 21:20:05,474 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/table/eval +2025-01-03 21:20:05,554 INFO wandb-upload_2:1975487 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpw9iems78 +2025-01-03 21:20:05,593 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 21:20:05,602 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:20:05,652 INFO wandb-upload_1:1975487 [upload_job.py:push():130] Uploaded file /tmp/tmpo3rn3d09wandb/gfkii59r-media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-03 21:20:05,755 INFO wandb-upload_0:1975487 [upload_job.py:push():130] Uploaded file /tmp/tmpo3rn3d09wandb/w7asjmes-events.out.tfevents.1735956966.leonmkim-ROG-Strix-G15CS-G15CS.1975301.0 +2025-01-03 21:20:05,960 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:06,034 INFO SenderThread:1975487 [sender.py:send_request_log_artifact():1525] logged artifact run-nayavtp6-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzkwNDMwNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY3ODMwNg==', 'latestArtifact': None}} +2025-01-03 21:20:06,034 DEBUG SenderThread:1975487 [sender.py:send():379] send: files +2025-01-03 21:20:06,034 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json with policy now +2025-01-03 21:20:06,034 DEBUG SenderThread:1975487 [sender.py:send():379] send: history +2025-01-03 21:20:06,034 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:20:06,035 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:20:06,035 DEBUG SenderThread:1975487 [sender.py:send():379] send: history +2025-01-03 21:20:06,036 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:20:06,036 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:20:06,036 DEBUG SenderThread:1975487 [sender.py:send():379] send: telemetry +2025-01-03 21:20:06,036 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:20:06,038 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:20:06,038 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:20:06,038 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:20:06,038 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:20:06,039 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:20:06,039 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:20:06,039 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:20:06,039 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:20:06,040 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:20:06,040 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:20:06,040 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:20:06,040 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:20:06,040 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:20:06,041 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:20:06,041 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:20:06,041 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:20:06,041 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:20:06,042 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:20:06,042 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:20:06,042 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:20:06,042 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:20:06,043 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 21:20:06,309 INFO wandb-upload_2:1975487 [upload_job.py:push():130] Uploaded file /tmp/tmpo3rn3d09wandb/575q2s1s-media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-03 21:20:06,422 INFO wandb-upload_0:1975487 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpbc37cpza +2025-01-03 21:20:06,474 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/wandb-summary.json +2025-01-03 21:20:06,474 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/events.out.tfevents.1735956966.leonmkim-ROG-Strix-G15CS-G15CS.1975301.0 +2025-01-03 21:20:06,474 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/table/eval/total_success_rate_ci_6_afac0fa2248ad7b10763.table.json +2025-01-03 21:20:06,475 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/table/eval +2025-01-03 21:20:06,834 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:20:06,844 INFO SenderThread:1975487 [sender.py:send_request_log_artifact():1525] logged artifact run-nayavtp6-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzkwNDMxNQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY3ODMwNw==', 'latestArtifact': None}} +2025-01-03 21:20:06,844 DEBUG SenderThread:1975487 [sender.py:send():379] send: files +2025-01-03 21:20:06,844 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_afac0fa2248ad7b10763.table.json with policy now +2025-01-03 21:20:06,845 DEBUG SenderThread:1975487 [sender.py:send():379] send: history +2025-01-03 21:20:06,845 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:20:06,845 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:20:06,845 DEBUG SenderThread:1975487 [sender.py:send():379] send: stats +2025-01-03 21:20:06,846 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:20:06,961 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:07,025 INFO wandb-upload_1:1975487 [upload_job.py:push():130] Uploaded file /tmp/tmpo3rn3d09wandb/0eq29wsn-media/table/eval/total_success_rate_ci_6_afac0fa2248ad7b10763.table.json +2025-01-03 21:20:07,279 INFO wandb-upload_0:1975487 [upload_job.py:push():130] Uploaded file /tmp/tmpo3rn3d09wandb/57fy6k8g-media/table/eval/total_success_rate_ci_6_afac0fa2248ad7b10763.table.json +2025-01-03 21:20:07,475 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/wandb-summary.json +2025-01-03 21:20:07,475 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:20:07,961 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:07,964 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:20:08,961 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:09,961 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:10,289 DEBUG SenderThread:1975487 [sender.py:send():379] send: exit +2025-01-03 21:20:10,289 INFO SenderThread:1975487 [sender.py:send_exit():586] handling exit code: 0 +2025-01-03 21:20:10,290 INFO SenderThread:1975487 [sender.py:send_exit():588] handling runtime: 243 +2025-01-03 21:20:10,290 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:20:10,290 INFO SenderThread:1975487 [sender.py:send_exit():594] send defer +2025-01-03 21:20:10,290 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:20:10,290 INFO HandlerThread:1975487 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-03 21:20:10,290 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: defer +2025-01-03 21:20:10,290 INFO SenderThread:1975487 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-03 21:20:10,290 INFO SenderThread:1975487 [sender.py:transition_state():614] send defer: 1 +2025-01-03 21:20:10,291 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:20:10,291 INFO HandlerThread:1975487 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-03 21:20:10,291 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: defer +2025-01-03 21:20:10,291 INFO SenderThread:1975487 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-03 21:20:10,291 INFO SenderThread:1975487 [sender.py:transition_state():614] send defer: 2 +2025-01-03 21:20:10,291 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:20:10,291 INFO HandlerThread:1975487 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-03 21:20:10,291 INFO HandlerThread:1975487 [system_monitor.py:finish():203] Stopping system monitor +2025-01-03 21:20:10,291 INFO HandlerThread:1975487 [interfaces.py:finish():200] Joined cpu monitor +2025-01-03 21:20:10,292 INFO HandlerThread:1975487 [interfaces.py:finish():200] Joined disk monitor +2025-01-03 21:20:10,292 DEBUG SystemMonitor:1975487 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-03 21:20:10,292 DEBUG SystemMonitor:1975487 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-03 21:20:10,344 INFO HandlerThread:1975487 [interfaces.py:finish():200] Joined gpu monitor +2025-01-03 21:20:10,345 INFO HandlerThread:1975487 [interfaces.py:finish():200] Joined memory monitor +2025-01-03 21:20:10,345 INFO HandlerThread:1975487 [interfaces.py:finish():200] Joined network monitor +2025-01-03 21:20:10,345 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: defer +2025-01-03 21:20:10,345 INFO SenderThread:1975487 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-03 21:20:10,345 INFO SenderThread:1975487 [sender.py:transition_state():614] send defer: 3 +2025-01-03 21:20:10,345 DEBUG SenderThread:1975487 [sender.py:send():379] send: stats +2025-01-03 21:20:10,345 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:20:10,346 INFO HandlerThread:1975487 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-03 21:20:10,346 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: defer +2025-01-03 21:20:10,346 INFO SenderThread:1975487 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-03 21:20:10,346 INFO SenderThread:1975487 [sender.py:transition_state():614] send defer: 4 +2025-01-03 21:20:10,346 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:20:10,346 INFO HandlerThread:1975487 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-03 21:20:10,476 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/wandb-summary.json +2025-01-03 21:20:11,476 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:20:17,174 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:20:17,174 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:20:17,174 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 21:20:17,175 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: defer +2025-01-03 21:20:17,175 INFO SenderThread:1975487 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-03 21:20:17,175 INFO SenderThread:1975487 [sender.py:transition_state():614] send defer: 5 +2025-01-03 21:20:17,175 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:20:17,175 DEBUG SenderThread:1975487 [sender.py:send():379] send: history +2025-01-03 21:20:17,176 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:20:17,176 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:20:17,176 INFO HandlerThread:1975487 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-03 21:20:17,177 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:20:17,177 DEBUG SenderThread:1975487 [sender.py:send():379] send: summary +2025-01-03 21:20:17,177 INFO SenderThread:1975487 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:20:17,177 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: defer +2025-01-03 21:20:17,177 INFO SenderThread:1975487 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-03 21:20:17,177 INFO SenderThread:1975487 [sender.py:transition_state():614] send defer: 6 +2025-01-03 21:20:17,177 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:20:17,178 INFO HandlerThread:1975487 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-03 21:20:17,178 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: defer +2025-01-03 21:20:17,178 INFO SenderThread:1975487 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-03 21:20:17,181 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:20:17,296 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:20:17,315 INFO SenderThread:1975487 [sender.py:transition_state():614] send defer: 7 +2025-01-03 21:20:17,315 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:20:17,315 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:20:17,315 INFO HandlerThread:1975487 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-03 21:20:17,315 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: defer +2025-01-03 21:20:17,315 INFO SenderThread:1975487 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-03 21:20:17,479 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/config.yaml +2025-01-03 21:20:17,479 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/wandb-summary.json +2025-01-03 21:20:18,296 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:20:20,712 INFO SenderThread:1975487 [sender.py:transition_state():614] send defer: 8 +2025-01-03 21:20:20,712 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:20:20,712 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:20:20,712 INFO HandlerThread:1975487 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-03 21:20:20,712 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: defer +2025-01-03 21:20:20,712 INFO SenderThread:1975487 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-03 21:20:20,712 INFO SenderThread:1975487 [job_builder.py:build():440] Attempting to build job artifact +2025-01-03 21:20:20,715 INFO SenderThread:1975487 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-03 21:20:20,717 INFO SenderThread:1975487 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-03 21:20:20,729 INFO SenderThread:1975487 [sender.py:transition_state():614] send defer: 9 +2025-01-03 21:20:20,729 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:20:20,729 INFO HandlerThread:1975487 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-03 21:20:20,729 DEBUG SenderThread:1975487 [sender.py:send():379] send: artifact +2025-01-03 21:20:21,150 INFO wandb-upload_2:1975487 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpkky9i0ng +2025-01-03 21:20:21,151 INFO wandb-upload_1:1975487 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpideqz279 +2025-01-03 21:20:21,297 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:20:21,371 INFO wandb-upload_0:1975487 [upload_job.py:push():88] Uploaded file /tmp/tmp5cg23i_h/wandb-job.json +2025-01-03 21:20:21,480 INFO Thread-12 :1975487 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:20:21,942 INFO SenderThread:1975487 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNzkwNDU3OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNzg5OTM1NQ==', 'versionIndex': 409}}} +2025-01-03 21:20:21,942 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: defer +2025-01-03 21:20:21,942 INFO SenderThread:1975487 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-03 21:20:21,942 INFO SenderThread:1975487 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-03 21:20:22,481 INFO SenderThread:1975487 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files +2025-01-03 21:20:22,481 INFO SenderThread:1975487 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/config.yaml config.yaml +2025-01-03 21:20:22,481 INFO SenderThread:1975487 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/wandb-summary.json wandb-summary.json +2025-01-03 21:20:22,482 INFO SenderThread:1975487 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/requirements.txt requirements.txt +2025-01-03 21:20:22,482 INFO SenderThread:1975487 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/wandb-metadata.json wandb-metadata.json +2025-01-03 21:20:22,482 INFO SenderThread:1975487 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/events.out.tfevents.1735956966.leonmkim-ROG-Strix-G15CS-G15CS.1975301.0 events.out.tfevents.1735956966.leonmkim-ROG-Strix-G15CS-G15CS.1975301.0 +2025-01-03 21:20:22,482 INFO SenderThread:1975487 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log output.log +2025-01-03 21:20:22,482 INFO SenderThread:1975487 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/diff.patch diff.patch +2025-01-03 21:20:22,487 INFO SenderThread:1975487 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-03 21:20:22,490 INFO SenderThread:1975487 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval/4_eval_4_8ec81cfff6351c2b5db4.mp4 media/videos/eval/4_eval_4_8ec81cfff6351c2b5db4.mp4 +2025-01-03 21:20:22,490 INFO SenderThread:1975487 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval/2_eval_2_a653d18806cfc6db0e07.mp4 media/videos/eval/2_eval_2_a653d18806cfc6db0e07.mp4 +2025-01-03 21:20:22,490 INFO SenderThread:1975487 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval/3_eval_3_a145a501509e034700f1.mp4 media/videos/eval/3_eval_3_a145a501509e034700f1.mp4 +2025-01-03 21:20:22,490 INFO SenderThread:1975487 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval/1_eval_1_91c0aab1f5aa1b260cf6.mp4 media/videos/eval/1_eval_1_91c0aab1f5aa1b260cf6.mp4 +2025-01-03 21:20:22,490 INFO SenderThread:1975487 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval/0_eval_0_29e1a738e40422f19100.mp4 media/videos/eval/0_eval_0_29e1a738e40422f19100.mp4 +2025-01-03 21:20:22,490 INFO SenderThread:1975487 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/table/eval/total_success_rate_ci_6_afac0fa2248ad7b10763.table.json media/table/eval/total_success_rate_ci_6_afac0fa2248ad7b10763.table.json +2025-01-03 21:20:22,491 INFO SenderThread:1975487 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-03 21:20:22,491 INFO SenderThread:1975487 [sender.py:transition_state():614] send defer: 10 +2025-01-03 21:20:22,491 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:20:22,491 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:20:22,491 INFO HandlerThread:1975487 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-03 21:20:22,491 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: defer +2025-01-03 21:20:22,491 INFO SenderThread:1975487 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-03 21:20:22,491 INFO SenderThread:1975487 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 21:20:22,670 INFO wandb-upload_0:1975487 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/requirements.txt +2025-01-03 21:20:22,678 INFO wandb-upload_2:1975487 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/config.yaml +2025-01-03 21:20:22,704 INFO wandb-upload_1:1975487 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/wandb-summary.json +2025-01-03 21:20:22,754 INFO wandb-upload_4:1975487 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/files/output.log +2025-01-03 21:20:22,770 INFO wandb-upload_3:1975487 [upload_job.py:push():130] Uploaded file /tmp/tmpo3rn3d09wandb/h0tghdfr-events.out.tfevents.1735956966.leonmkim-ROG-Strix-G15CS-G15CS.1975301.0 +2025-01-03 21:20:22,971 INFO Thread-11 (_thread_body):1975487 [sender.py:transition_state():614] send defer: 11 +2025-01-03 21:20:22,971 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:20:22,971 INFO HandlerThread:1975487 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-03 21:20:22,971 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: defer +2025-01-03 21:20:22,971 INFO SenderThread:1975487 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-03 21:20:22,971 INFO SenderThread:1975487 [file_pusher.py:join():175] waiting for file pusher +2025-01-03 21:20:22,972 INFO SenderThread:1975487 [sender.py:transition_state():614] send defer: 12 +2025-01-03 21:20:22,972 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:20:22,972 INFO HandlerThread:1975487 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-03 21:20:22,973 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: defer +2025-01-03 21:20:22,973 INFO SenderThread:1975487 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-03 21:20:22,973 INFO SenderThread:1975487 [file_stream.py:finish():601] file stream finish called +2025-01-03 21:20:23,050 INFO SenderThread:1975487 [file_stream.py:finish():605] file stream finish is done +2025-01-03 21:20:23,050 INFO SenderThread:1975487 [sender.py:transition_state():614] send defer: 13 +2025-01-03 21:20:23,050 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:20:23,050 INFO HandlerThread:1975487 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-03 21:20:23,050 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: defer +2025-01-03 21:20:23,050 INFO SenderThread:1975487 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-03 21:20:23,050 INFO SenderThread:1975487 [sender.py:transition_state():614] send defer: 14 +2025-01-03 21:20:23,050 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:20:23,051 INFO HandlerThread:1975487 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-03 21:20:23,051 DEBUG SenderThread:1975487 [sender.py:send():379] send: final +2025-01-03 21:20:23,051 DEBUG SenderThread:1975487 [sender.py:send():379] send: footer +2025-01-03 21:20:23,051 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: defer +2025-01-03 21:20:23,051 INFO SenderThread:1975487 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-03 21:20:23,051 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:20:23,052 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: server_info +2025-01-03 21:20:23,052 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:20:23,052 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:20:23,052 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: get_summary +2025-01-03 21:20:23,052 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: server_info +2025-01-03 21:20:23,053 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-03 21:20:23,055 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:20:23,115 DEBUG SenderThread:1975487 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:20:23,115 INFO MainThread:1975487 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-03 21:20:23,116 INFO MainThread:1975487 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-03 21:20:23,116 INFO MainThread:1975487 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-03 21:20:23,116 DEBUG HandlerThread:1975487 [handler.py:handle_request():158] handle_request: shutdown +2025-01-03 21:20:23,116 INFO HandlerThread:1975487 [handler.py:finish():882] shutting down handler +2025-01-03 21:20:24,052 INFO WriterThread:1975487 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/run-nayavtp6.wandb +2025-01-03 21:20:24,115 INFO SenderThread:1975487 [sender.py:finish():1615] shutting down sender +2025-01-03 21:20:24,115 INFO SenderThread:1975487 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 21:20:24,115 INFO SenderThread:1975487 [file_pusher.py:join():175] waiting for file pusher diff --git a/nayavtp6/wandb/run-20250103_211606-nayavtp6/logs/debug.log b/nayavtp6/wandb/run-20250103_211606-nayavtp6/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..bdb88a623e859512d4ab61bbdd1a3c887f37276c --- /dev/null +++ b/nayavtp6/wandb/run-20250103_211606-nayavtp6/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-03 21:16:06,125 INFO MainThread:1975301 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-03 21:16:06,125 INFO MainThread:1975301 [wandb_setup.py:_flush():76] Configure stats pid to 1975301 +2025-01-03 21:16:06,125 INFO MainThread:1975301 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-03 21:16:06,125 INFO MainThread:1975301 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/settings +2025-01-03 21:16:06,125 INFO MainThread:1975301 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-03 21:16:06,125 INFO MainThread:1975301 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-03 21:16:06,125 INFO MainThread:1975301 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-03 21:16:06,125 INFO MainThread:1975301 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-03 21:16:06,125 INFO MainThread:1975301 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/logs/debug.log +2025-01-03 21:16:06,125 INFO MainThread:1975301 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/wandb/run-20250103_211606-nayavtp6/logs/debug-internal.log +2025-01-03 21:16:06,125 INFO MainThread:1975301 [wandb_init.py:init():569] calling init triggers +2025-01-03 21:16:06,125 INFO MainThread:1975301 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1926_1', 'wandb_notes': '1926_1_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6'} +2025-01-03 21:16:06,125 INFO MainThread:1975301 [wandb_init.py:init():619] starting backend +2025-01-03 21:16:06,126 INFO MainThread:1975301 [wandb_init.py:init():623] setting up manager +2025-01-03 21:16:06,129 INFO MainThread:1975301 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-03 21:16:06,130 INFO MainThread:1975301 [wandb_init.py:init():631] backend started and connected +2025-01-03 21:16:06,152 INFO MainThread:1975301 [wandb_init.py:init():720] updated telemetry +2025-01-03 21:16:06,158 INFO MainThread:1975301 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-03 21:16:06,412 INFO MainThread:1975301 [wandb_run.py:_on_init():2435] communicating current version +2025-01-03 21:16:06,486 INFO MainThread:1975301 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-03 21:16:06,486 INFO MainThread:1975301 [wandb_init.py:init():804] starting run threads in backend +2025-01-03 21:16:06,831 INFO MainThread:1975301 [wandb_run.py:_console_start():2413] atexit reg +2025-01-03 21:16:06,831 INFO MainThread:1975301 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-03 21:16:06,831 INFO MainThread:1975301 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-03 21:16:06,831 INFO MainThread:1975301 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-03 21:16:06,833 INFO MainThread:1975301 [wandb_init.py:init():847] run started, returning control to user process +2025-01-03 21:16:06,833 INFO MainThread:1975301 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nayavtp6/tb, True +2025-01-03 21:16:11,355 INFO MainThread:1975301 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'} +2025-01-03 21:20:24,117 WARNING MsgRouterThr:1975301 [router.py:message_loop():77] message_loop has been closed diff --git a/nayavtp6/wandb/run-20250103_211606-nayavtp6/run-nayavtp6.wandb b/nayavtp6/wandb/run-20250103_211606-nayavtp6/run-nayavtp6.wandb new file mode 100644 index 0000000000000000000000000000000000000000..58957eb6845df0b8d9c5cbd26d745f89f9c2b42a Binary files /dev/null and b/nayavtp6/wandb/run-20250103_211606-nayavtp6/run-nayavtp6.wandb differ diff --git a/nl3lmlc3/.hydra/config.yaml b/nl3lmlc3/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5614a531a25d335cfde80ac1396030650a108114 --- /dev/null +++ b/nl3lmlc3/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1926_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/nl3lmlc3/.hydra/hydra.yaml b/nl3lmlc3/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ed9e8b4b52e086faa7dac190c8f029f0b56b312b --- /dev/null +++ b/nl3lmlc3/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/nl3lmlc3/.hydra/overrides.yaml b/nl3lmlc3/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/nl3lmlc3/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/nl3lmlc3/episode_rosbags/aligned_depth_to_color_K.npy b/nl3lmlc3/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/nl3lmlc3/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/nl3lmlc3/episode_rosbags/color_K.npy b/nl3lmlc3/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/nl3lmlc3/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/nl3lmlc3/episode_rosbags/episode_0_2025-01-04-21-34-30.bag b/nl3lmlc3/episode_rosbags/episode_0_2025-01-04-21-34-30.bag new file mode 100644 index 0000000000000000000000000000000000000000..49ffcd847e41c26c49a4498cf1df10dc34549452 --- /dev/null +++ b/nl3lmlc3/episode_rosbags/episode_0_2025-01-04-21-34-30.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e532224a871c0e97b77d3a676e936a448d0f2773ef64979598ce281b4c104ad +size 2608936731 diff --git a/nl3lmlc3/episode_rosbags/episode_2_2025-01-04-21-36-11.bag b/nl3lmlc3/episode_rosbags/episode_2_2025-01-04-21-36-11.bag new file mode 100644 index 0000000000000000000000000000000000000000..beada25d9bdf026c6f516852d018bc2f139fcad1 --- /dev/null +++ b/nl3lmlc3/episode_rosbags/episode_2_2025-01-04-21-36-11.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8cea5c00f668be07d015da75415056fee5e56108538c6335eefd8f32a8d468c +size 2606859783 diff --git a/nl3lmlc3/episode_rosbags/episode_3_2025-01-04-21-37-04.bag b/nl3lmlc3/episode_rosbags/episode_3_2025-01-04-21-37-04.bag new file mode 100644 index 0000000000000000000000000000000000000000..3ad6d4f01e46a4e1da33ab2c74024c0769b35782 --- /dev/null +++ b/nl3lmlc3/episode_rosbags/episode_3_2025-01-04-21-37-04.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fc6fcc2c857c1c6783bfca2be63c6bce4f2839483030ff7cdbd090d7ed62847 +size 2608748328 diff --git a/nl3lmlc3/eval_video/0_eval.mp4 b/nl3lmlc3/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..606913388abb1b25dc3ff293f1ddfd57e7831126 --- /dev/null +++ b/nl3lmlc3/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16d80c9a1dc6763cf29a3b90accf6875237b314055606884f38bf6a1c8055147 +size 1450694 diff --git a/nl3lmlc3/eval_video/1_eval.mp4 b/nl3lmlc3/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..51d3c54846475c1c4cefd67c60a98974e23437c6 --- /dev/null +++ b/nl3lmlc3/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9d9568f97ca21d8f63ad8b430332b1dddc7fc31a5f8686832b15d9d2c0adfc8 +size 1446649 diff --git a/nl3lmlc3/eval_video/2_eval.mp4 b/nl3lmlc3/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39238d85726e63fd3422dc1960d752fad8ab679b --- /dev/null +++ b/nl3lmlc3/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4982ced87e18fd534ea7632a98a9e9397fbdc9ace6e94d61111a1cc8a488e7f5 +size 1464960 diff --git a/nl3lmlc3/eval_video/3_eval.mp4 b/nl3lmlc3/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8a2720849b9d37e801068b5ee135e41366c4d3f9 --- /dev/null +++ b/nl3lmlc3/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8c539eab0fc6faa4df8dd4c58806f53068a10298a261b6f7a0016f5798a250f +size 1461798 diff --git a/nl3lmlc3/eval_video/4_eval.mp4 b/nl3lmlc3/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..515fbad906e22740a2026dcaadb9ee90f77854a8 --- /dev/null +++ b/nl3lmlc3/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06b29357871c19fff932800d727eabeeea718317e8e920b418420132fa692631 +size 1469723 diff --git a/nl3lmlc3/tb/events.out.tfevents.1736044453.leonmkim-ROG-Strix-G15CS-G15CS.2028630.0 b/nl3lmlc3/tb/events.out.tfevents.1736044453.leonmkim-ROG-Strix-G15CS-G15CS.2028630.0 new file mode 100644 index 0000000000000000000000000000000000000000..b31a9e4a00a2cc8552e967455ffd5d097b011611 --- /dev/null +++ b/nl3lmlc3/tb/events.out.tfevents.1736044453.leonmkim-ROG-Strix-G15CS-G15CS.2028630.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94649f81fc2a3ddbd63ac1ffa0ec6fa48a2f99844cd8464c6b2668f1c744ef7f +size 6539 diff --git a/nl3lmlc3/wandb/debug-internal.log b/nl3lmlc3/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..ff20444d409656fb4672fc7d73c8dc64616e0625 --- /dev/null +++ b/nl3lmlc3/wandb/debug-internal.log @@ -0,0 +1,789 @@ +2025-01-04 21:34:13,052 INFO StreamThr :2028802 [internal.py:wandb_internal():85] W&B internal server running at pid: 2028802, started at: 2025-01-04 21:34:13.052159 +2025-01-04 21:34:13,054 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status +2025-01-04 21:34:13,065 INFO WriterThread:2028802 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/run-nl3lmlc3.wandb +2025-01-04 21:34:13,065 DEBUG SenderThread:2028802 [sender.py:send():379] send: header +2025-01-04 21:34:13,074 DEBUG SenderThread:2028802 [sender.py:send():379] send: run +2025-01-04 21:34:13,300 INFO SenderThread:2028802 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files +2025-01-04 21:34:13,300 INFO SenderThread:2028802 [sender.py:_start_run_threads():1188] run started: nl3lmlc3 with start time 1736044453.052613 +2025-01-04 21:34:13,325 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: check_version +2025-01-04 21:34:13,326 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: check_version +2025-01-04 21:34:13,396 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: run_start +2025-01-04 21:34:13,401 DEBUG HandlerThread:2028802 [system_info.py:__init__():26] System info init +2025-01-04 21:34:13,401 DEBUG HandlerThread:2028802 [system_info.py:__init__():41] System info init done +2025-01-04 21:34:13,401 INFO HandlerThread:2028802 [system_monitor.py:start():194] Starting system monitor +2025-01-04 21:34:13,401 INFO SystemMonitor:2028802 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-04 21:34:13,402 INFO HandlerThread:2028802 [system_monitor.py:probe():214] Collecting system info +2025-01-04 21:34:13,402 INFO SystemMonitor:2028802 [interfaces.py:start():188] Started cpu monitoring +2025-01-04 21:34:13,403 INFO SystemMonitor:2028802 [interfaces.py:start():188] Started disk monitoring +2025-01-04 21:34:13,404 INFO SystemMonitor:2028802 [interfaces.py:start():188] Started gpu monitoring +2025-01-04 21:34:13,404 INFO SystemMonitor:2028802 [interfaces.py:start():188] Started memory monitoring +2025-01-04 21:34:13,405 INFO SystemMonitor:2028802 [interfaces.py:start():188] Started network monitoring +2025-01-04 21:34:13,414 DEBUG HandlerThread:2028802 [system_info.py:probe():152] Probing system +2025-01-04 21:34:13,416 DEBUG HandlerThread:2028802 [system_info.py:_probe_git():137] Probing git +2025-01-04 21:34:13,421 DEBUG HandlerThread:2028802 [system_info.py:_probe_git():145] Probing git done +2025-01-04 21:34:13,421 DEBUG HandlerThread:2028802 [system_info.py:probe():200] Probing system done +2025-01-04 21:34:13,421 DEBUG HandlerThread:2028802 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-05T02:34:13.414976', 'startedAt': '2025-01-05T02:34:13.042598', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '02cf07d901f3d8187a302cd51c9faa80321417ee'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.683, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.919, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4497.128, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.97, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 688.5316734313965}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-04 21:34:13,421 INFO HandlerThread:2028802 [system_monitor.py:probe():224] Finished collecting system info +2025-01-04 21:34:13,421 INFO HandlerThread:2028802 [system_monitor.py:probe():227] Publishing system info +2025-01-04 21:34:13,421 DEBUG HandlerThread:2028802 [system_info.py:_save_code():44] Saving code +2025-01-04 21:34:13,427 DEBUG HandlerThread:2028802 [system_info.py:_save_code():67] Saving code done +2025-01-04 21:34:13,427 DEBUG HandlerThread:2028802 [system_info.py:_save_patches():84] Saving git patches +2025-01-04 21:34:13,488 DEBUG HandlerThread:2028802 [system_info.py:_save_patches():126] Saving git patches done +2025-01-04 21:34:13,489 INFO HandlerThread:2028802 [system_monitor.py:probe():229] Finished publishing system info +2025-01-04 21:34:13,492 DEBUG SenderThread:2028802 [sender.py:send():379] send: files +2025-01-04 21:34:13,492 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-04 21:34:13,492 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-04 21:34:13,492 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-04 21:34:13,737 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: python_packages +2025-01-04 21:34:13,737 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: python_packages +2025-01-04 21:34:13,738 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:34:13,738 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:34:13,740 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:13,741 INFO HandlerThread:2028802 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/tb" + save: true +} +control { + relay_id: "3a7195e3c3d043c5990cffbb2121e77d" +} +_info { + stream_id: "nl3lmlc3" +} + +2025-01-04 21:34:13,741 DEBUG HandlerThread:2028802 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-04 21:34:13,760 INFO wandb-upload_2:2028802 [upload_job.py:push():130] Uploaded file /tmp/tmp6wgcj3uiwandb/dxoriorl-diff.patch +2025-01-04 21:34:13,766 INFO wandb-upload_1:2028802 [upload_job.py:push():130] Uploaded file /tmp/tmp6wgcj3uiwandb/l6vnklxd-code/FISH/eval_robot.py +2025-01-04 21:34:13,772 INFO wandb-upload_0:2028802 [upload_job.py:push():130] Uploaded file /tmp/tmp6wgcj3uiwandb/qut0tp3p-wandb-metadata.json +2025-01-04 21:34:13,904 DEBUG SenderThread:2028802 [sender.py:send():379] send: telemetry +2025-01-04 21:34:13,904 DEBUG SenderThread:2028802 [sender.py:send():379] send: tbrecord +2025-01-04 21:34:13,904 DEBUG SenderThread:2028802 [sender.py:send():379] send: files +2025-01-04 21:34:13,904 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file events.out.tfevents.1736044453.leonmkim-ROG-Strix-G15CS-G15CS.2028630.0 with policy live +2025-01-04 21:34:14,107 INFO wandb-upload_2:2028802 [upload_job.py:push():130] Uploaded file /tmp/tmp6wgcj3uiwandb/tsbq0c5s-events.out.tfevents.1736044453.leonmkim-ROG-Strix-G15CS-G15CS.2028630.0 +2025-01-04 21:34:14,301 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/wandb-metadata.json +2025-01-04 21:34:14,302 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/requirements.txt +2025-01-04 21:34:14,302 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/code/FISH/eval_robot.py +2025-01-04 21:34:14,302 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:34:14,302 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/diff.patch +2025-01-04 21:34:14,302 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/events.out.tfevents.1736044453.leonmkim-ROG-Strix-G15CS-G15CS.2028630.0 +2025-01-04 21:34:14,302 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/code/FISH +2025-01-04 21:34:14,302 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/code +2025-01-04 21:34:14,738 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:15,738 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:16,301 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:34:16,738 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:17,739 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:18,141 DEBUG SenderThread:2028802 [sender.py:send():379] send: config +2025-01-04 21:34:18,141 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:34:18,739 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:19,739 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:20,302 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:34:20,739 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:21,739 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:22,740 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:23,738 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:34:23,739 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:24,303 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:34:24,740 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:25,740 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:26,740 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:27,740 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:28,304 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:34:28,738 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:34:28,738 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:34:28,740 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:28,889 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:34:29,740 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:30,305 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:34:30,740 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:31,741 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:32,305 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:34:32,741 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:33,743 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:34,738 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:34:34,743 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:35,743 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:36,307 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:34:36,743 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:37,743 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:38,743 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:39,739 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:34:39,743 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:40,743 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:41,743 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:42,744 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:43,738 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:34:43,739 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:34:43,781 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:44,744 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:44,800 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:34:45,309 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/config.yaml +2025-01-04 21:34:45,744 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:46,744 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:47,744 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:48,744 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:49,744 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:50,740 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:34:50,744 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:51,745 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:52,745 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:53,745 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:54,745 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:55,740 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:34:55,745 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:56,747 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:57,746 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:58,738 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:34:58,739 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:34:58,781 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:59,746 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:00,747 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:00,859 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:35:01,747 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:02,747 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:03,747 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:04,748 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:05,748 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:06,328 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:35:06,533 DEBUG SenderThread:2028802 [sender.py:send():379] send: files +2025-01-04 21:35:06,533 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_16d80c9a1dc6763cf29a.mp4 with policy now +2025-01-04 21:35:06,534 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:35:06,535 DEBUG SenderThread:2028802 [sender.py:send():379] send: history +2025-01-04 21:35:06,535 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:35:06,536 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:35:06,748 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:06,773 INFO wandb-upload_0:2028802 [upload_job.py:push():130] Uploaded file /tmp/tmp6wgcj3uiwandb/h48q6q0p-media/videos/eval/0_eval_0_16d80c9a1dc6763cf29a.mp4 +2025-01-04 21:35:07,315 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/wandb-summary.json +2025-01-04 21:35:07,315 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval/0_eval_0_16d80c9a1dc6763cf29a.mp4 +2025-01-04 21:35:07,315 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media +2025-01-04 21:35:07,315 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos +2025-01-04 21:35:07,315 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval +2025-01-04 21:35:07,579 INFO wandb-upload_2:2028802 [upload_job.py:push():130] Uploaded file /tmp/tmp6wgcj3uiwandb/2qhyr499-media/videos/eval/0_eval_0_16d80c9a1dc6763cf29a.mp4 +2025-01-04 21:35:07,748 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:08,315 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:35:08,748 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:09,749 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:10,749 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:11,749 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:12,193 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:35:12,316 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:35:12,749 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:13,405 DEBUG SystemMonitor:2028802 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-04 21:35:13,406 DEBUG SenderThread:2028802 [sender.py:send():379] send: stats +2025-01-04 21:35:13,739 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:35:13,739 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:35:13,749 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:14,317 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:35:14,749 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:15,749 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:16,749 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:17,750 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:17,843 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:35:18,749 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:19,750 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:20,319 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/events.out.tfevents.1736044453.leonmkim-ROG-Strix-G15CS-G15CS.2028630.0 +2025-01-04 21:35:20,319 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:35:20,554 INFO wandb-upload_0:2028802 [upload_job.py:push():130] Uploaded file /tmp/tmp6wgcj3uiwandb/j5dil0ce-events.out.tfevents.1736044453.leonmkim-ROG-Strix-G15CS-G15CS.2028630.0 +2025-01-04 21:35:20,750 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:21,750 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:22,320 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:35:22,751 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:23,238 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:35:23,754 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:24,321 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:35:24,751 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:25,751 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:26,321 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:35:26,752 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:27,752 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:28,739 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:35:28,739 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:35:28,785 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:28,896 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:35:29,752 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:30,752 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:31,752 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:32,752 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:33,752 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:34,749 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:35:34,752 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:35,765 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:36,753 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:37,753 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:38,326 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:35:38,753 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:39,749 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:35:39,753 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:40,754 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:41,754 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:42,754 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:43,406 DEBUG SenderThread:2028802 [sender.py:send():379] send: stats +2025-01-04 21:35:43,739 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:35:43,739 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:35:43,781 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:44,754 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:44,860 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:35:45,754 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:46,754 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:47,754 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:48,754 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:49,755 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:50,749 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:35:50,755 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:51,755 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:52,755 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:53,755 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:54,755 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:55,749 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:35:55,755 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:56,755 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:57,755 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:58,739 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:35:58,739 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:35:58,759 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:59,153 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:35:59,153 DEBUG SenderThread:2028802 [sender.py:send():379] send: files +2025-01-04 21:35:59,154 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_c9d9568f97ca21d8f63a.mp4 with policy now +2025-01-04 21:35:59,154 DEBUG SenderThread:2028802 [sender.py:send():379] send: history +2025-01-04 21:35:59,154 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:35:59,160 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:35:59,332 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/wandb-summary.json +2025-01-04 21:35:59,332 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval/1_eval_1_c9d9568f97ca21d8f63a.mp4 +2025-01-04 21:35:59,333 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval +2025-01-04 21:35:59,386 INFO wandb-upload_2:2028802 [upload_job.py:push():130] Uploaded file /tmp/tmp6wgcj3uiwandb/5v7yfz83-media/videos/eval/1_eval_1_c9d9568f97ca21d8f63a.mp4 +2025-01-04 21:35:59,637 INFO wandb-upload_0:2028802 [upload_job.py:push():130] Uploaded file /tmp/tmp6wgcj3uiwandb/3cfe767r-media/videos/eval/1_eval_1_c9d9568f97ca21d8f63a.mp4 +2025-01-04 21:35:59,758 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:00,333 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:36:00,760 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:01,166 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:36:01,762 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:02,762 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:03,771 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:04,334 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:36:04,766 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:05,764 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:06,335 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:36:06,541 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:36:06,764 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:07,764 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:08,152 DEBUG SenderThread:2028802 [sender.py:send():379] send: history +2025-01-04 21:36:08,152 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:36:08,153 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:36:08,336 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/wandb-summary.json +2025-01-04 21:36:08,336 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/events.out.tfevents.1736044453.leonmkim-ROG-Strix-G15CS-G15CS.2028630.0 +2025-01-04 21:36:08,336 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:36:08,765 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:09,764 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:10,336 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:36:10,764 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:11,595 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:36:11,779 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:12,337 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:36:12,772 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:13,407 DEBUG SenderThread:2028802 [sender.py:send():379] send: stats +2025-01-04 21:36:13,739 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:36:13,739 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:36:13,781 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:14,338 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:36:14,772 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:15,773 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:16,773 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:16,898 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:36:17,773 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:18,773 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:19,773 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:20,773 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:21,773 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:22,221 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:36:22,773 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:23,774 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:24,341 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:36:24,774 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:25,774 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:26,342 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:36:26,774 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:27,774 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:27,865 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:36:28,739 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:36:28,739 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:36:28,781 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:29,774 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:30,774 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:31,775 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:32,776 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:32,895 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:36:33,776 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:34,775 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:35,776 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:36,345 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:36:36,776 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:37,778 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:38,752 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:36:38,777 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:39,777 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:40,777 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:41,777 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:42,778 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:43,407 DEBUG SenderThread:2028802 [sender.py:send():379] send: stats +2025-01-04 21:36:43,739 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:36:43,740 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:36:43,781 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:43,896 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:36:44,778 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:45,778 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:46,781 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:47,781 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:47,797 DEBUG SenderThread:2028802 [sender.py:send():379] send: files +2025-01-04 21:36:47,797 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_4982ced87e18fd534ea7.mp4 with policy now +2025-01-04 21:36:47,841 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:36:47,845 DEBUG SenderThread:2028802 [sender.py:send():379] send: history +2025-01-04 21:36:47,845 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:36:47,851 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:36:48,039 INFO wandb-upload_2:2028802 [upload_job.py:push():130] Uploaded file /tmp/tmp6wgcj3uiwandb/vgauh208-media/videos/eval/2_eval_2_4982ced87e18fd534ea7.mp4 +2025-01-04 21:36:48,349 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/wandb-summary.json +2025-01-04 21:36:48,349 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:36:48,349 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval/2_eval_2_4982ced87e18fd534ea7.mp4 +2025-01-04 21:36:48,350 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval +2025-01-04 21:36:48,618 INFO wandb-upload_0:2028802 [upload_job.py:push():130] Uploaded file /tmp/tmp6wgcj3uiwandb/j3osszjo-media/videos/eval/2_eval_2_4982ced87e18fd534ea7.mp4 +2025-01-04 21:36:48,790 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:49,758 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:36:49,785 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:50,350 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:36:50,782 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:51,785 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:52,350 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:36:52,790 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:53,793 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:54,787 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:55,377 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:36:55,787 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:56,352 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:36:56,792 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:57,791 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:58,744 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:36:58,745 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:36:58,795 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:59,796 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:00,798 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:00,827 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:37:00,993 DEBUG SenderThread:2028802 [sender.py:send():379] send: history +2025-01-04 21:37:00,993 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:37:00,994 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:37:01,353 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/wandb-summary.json +2025-01-04 21:37:01,354 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/events.out.tfevents.1736044453.leonmkim-ROG-Strix-G15CS-G15CS.2028630.0 +2025-01-04 21:37:01,575 INFO wandb-upload_2:2028802 [upload_job.py:push():130] Uploaded file /tmp/tmp6wgcj3uiwandb/u47fpcpp-events.out.tfevents.1736044453.leonmkim-ROG-Strix-G15CS-G15CS.2028630.0 +2025-01-04 21:37:01,794 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:02,354 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:37:02,799 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:03,798 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:04,355 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:37:04,796 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:05,796 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:05,896 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:37:06,355 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:37:06,796 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:07,796 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:08,797 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:09,797 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:10,797 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:11,756 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:37:11,797 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:12,797 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:13,408 DEBUG SenderThread:2028802 [sender.py:send():379] send: stats +2025-01-04 21:37:13,741 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:37:13,741 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:37:13,797 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:14,797 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:15,797 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:16,798 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:16,816 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:37:17,798 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:18,798 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:19,798 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:20,798 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:21,798 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:22,756 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:37:22,798 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:23,798 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:24,799 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:25,799 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:26,799 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:27,757 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:37:27,799 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:28,741 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:37:28,741 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:37:28,799 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:29,799 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:30,799 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:31,800 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:32,800 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:32,848 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:37:33,800 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:34,800 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:35,800 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:36,800 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:37,800 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:38,757 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:37:38,800 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:39,804 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:40,366 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:37:40,647 DEBUG SenderThread:2028802 [sender.py:send():379] send: files +2025-01-04 21:37:40,647 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_e8c539eab0fc6faa4df8.mp4 with policy now +2025-01-04 21:37:40,650 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:37:40,650 DEBUG SenderThread:2028802 [sender.py:send():379] send: history +2025-01-04 21:37:40,651 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:37:40,739 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:37:40,805 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:40,893 INFO wandb-upload_0:2028802 [upload_job.py:push():130] Uploaded file /tmp/tmp6wgcj3uiwandb/xc5rptpf-media/videos/eval/3_eval_3_e8c539eab0fc6faa4df8.mp4 +2025-01-04 21:37:41,367 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/wandb-summary.json +2025-01-04 21:37:41,367 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval/3_eval_3_e8c539eab0fc6faa4df8.mp4 +2025-01-04 21:37:41,367 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval +2025-01-04 21:37:41,660 INFO wandb-upload_2:2028802 [upload_job.py:push():130] Uploaded file /tmp/tmp6wgcj3uiwandb/chdq7vcb-media/videos/eval/3_eval_3_e8c539eab0fc6faa4df8.mp4 +2025-01-04 21:37:41,817 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:42,367 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:37:42,812 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:43,408 DEBUG SenderThread:2028802 [sender.py:send():379] send: stats +2025-01-04 21:37:43,748 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:37:43,748 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:37:43,814 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:43,894 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:37:44,822 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:45,813 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:46,369 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:37:46,813 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:47,814 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:48,369 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:37:48,815 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:49,762 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:37:49,817 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:50,818 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:51,820 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:52,821 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:53,371 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/events.out.tfevents.1736044453.leonmkim-ROG-Strix-G15CS-G15CS.2028630.0 +2025-01-04 21:37:53,820 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:54,056 DEBUG SenderThread:2028802 [sender.py:send():379] send: history +2025-01-04 21:37:54,056 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:37:54,058 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:37:54,371 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/wandb-summary.json +2025-01-04 21:37:54,372 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:37:54,817 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:55,237 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:37:55,820 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:56,372 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:37:56,821 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:57,819 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:58,373 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:37:58,742 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:37:58,742 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:37:58,819 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:59,819 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:00,819 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:00,871 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:38:01,820 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:02,820 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:03,820 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:04,820 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:05,820 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:06,157 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:38:06,376 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:38:06,820 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:07,820 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:08,820 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:09,821 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:10,821 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:11,760 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:38:11,821 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:12,821 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:13,409 DEBUG SenderThread:2028802 [sender.py:send():379] send: stats +2025-01-04 21:38:13,742 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:38:13,743 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:38:13,821 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:14,821 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:15,821 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:16,821 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:16,840 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:38:17,822 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:18,822 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:19,822 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:20,822 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:21,822 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:22,760 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:38:22,822 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:23,822 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:24,822 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:25,822 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:26,823 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:27,760 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:38:27,823 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:28,742 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:38:28,743 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:38:28,823 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:29,823 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:30,823 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:31,823 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:32,385 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:38:32,833 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:32,881 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:38:33,442 DEBUG SenderThread:2028802 [sender.py:send():379] send: files +2025-01-04 21:38:33,442 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_06b29357871c19fff932.mp4 with policy now +2025-01-04 21:38:33,445 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:38:33,446 DEBUG SenderThread:2028802 [sender.py:send():379] send: history +2025-01-04 21:38:33,446 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:38:33,448 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:38:33,666 INFO wandb-upload_0:2028802 [upload_job.py:push():130] Uploaded file /tmp/tmp6wgcj3uiwandb/vt6hrxwp-media/videos/eval/4_eval_4_06b29357871c19fff932.mp4 +2025-01-04 21:38:33,823 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:34,386 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/wandb-summary.json +2025-01-04 21:38:34,386 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:38:34,386 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval/4_eval_4_06b29357871c19fff932.mp4 +2025-01-04 21:38:34,387 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval +2025-01-04 21:38:34,658 INFO wandb-upload_2:2028802 [upload_job.py:push():130] Uploaded file /tmp/tmp6wgcj3uiwandb/52pb5k3q-media/videos/eval/4_eval_4_06b29357871c19fff932.mp4 +2025-01-04 21:38:34,830 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:35,827 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:36,386 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:38:36,834 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:37,828 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:38,117 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:38:38,387 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:38:38,836 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:39,840 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:40,841 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:41,846 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:42,388 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:38:42,848 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:43,417 DEBUG SenderThread:2028802 [sender.py:send():379] send: stats +2025-01-04 21:38:43,417 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:38:43,751 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:38:43,751 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:38:43,852 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:43,974 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 21:38:43,974 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 21:38:44,125 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:38:44,389 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/events.out.tfevents.1736044453.leonmkim-ROG-Strix-G15CS-G15CS.2028630.0 +2025-01-04 21:38:44,389 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:38:44,390 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-04 21:38:44,390 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media +2025-01-04 21:38:44,390 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/table/eval +2025-01-04 21:38:44,390 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/table +2025-01-04 21:38:44,478 INFO wandb-upload_0:2028802 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmprpart9pb +2025-01-04 21:38:44,561 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 21:38:44,570 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:38:44,628 INFO wandb-upload_1:2028802 [upload_job.py:push():130] Uploaded file /tmp/tmp6wgcj3uiwandb/mkln1ksu-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-04 21:38:44,642 INFO wandb-upload_2:2028802 [upload_job.py:push():130] Uploaded file /tmp/tmp6wgcj3uiwandb/vwjt6kxp-events.out.tfevents.1736044453.leonmkim-ROG-Strix-G15CS-G15CS.2028630.0 +2025-01-04 21:38:44,842 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:44,869 INFO SenderThread:2028802 [sender.py:send_request_log_artifact():1525] logged artifact run-nl3lmlc3-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE1OTQ1NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1NzUzNg==', 'latestArtifact': None}} +2025-01-04 21:38:44,869 DEBUG SenderThread:2028802 [sender.py:send():379] send: files +2025-01-04 21:38:44,869 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now +2025-01-04 21:38:44,870 DEBUG SenderThread:2028802 [sender.py:send():379] send: history +2025-01-04 21:38:44,870 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:38:44,870 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:38:44,870 DEBUG SenderThread:2028802 [sender.py:send():379] send: history +2025-01-04 21:38:44,871 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:38:44,874 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:38:44,874 DEBUG SenderThread:2028802 [sender.py:send():379] send: telemetry +2025-01-04 21:38:44,875 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:38:44,875 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:38:44,875 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:38:44,875 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:38:44,875 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:38:44,876 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:38:44,876 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:38:44,876 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:38:44,876 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:38:44,877 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:38:44,877 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:38:44,877 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:38:44,877 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:38:44,878 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:38:44,878 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:38:44,878 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:38:44,878 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:38:44,879 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:38:44,879 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:38:44,879 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:38:44,879 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:38:44,879 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:38:44,880 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 21:38:45,108 INFO wandb-upload_0:2028802 [upload_job.py:push():130] Uploaded file /tmp/tmp6wgcj3uiwandb/6shgp16c-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-04 21:38:45,265 INFO wandb-upload_2:2028802 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpjzr7cqer +2025-01-04 21:38:45,390 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/wandb-summary.json +2025-01-04 21:38:45,390 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/events.out.tfevents.1736044453.leonmkim-ROG-Strix-G15CS-G15CS.2028630.0 +2025-01-04 21:38:45,390 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/table/eval/total_success_rate_ci_6_3e9f0bce1c3a62e7f00b.table.json +2025-01-04 21:38:45,390 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/table/eval +2025-01-04 21:38:45,656 INFO SenderThread:2028802 [sender.py:send_request_log_artifact():1525] logged artifact run-nl3lmlc3-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE1OTQ2OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1NzUzNw==', 'latestArtifact': None}} +2025-01-04 21:38:45,656 DEBUG SenderThread:2028802 [sender.py:send():379] send: files +2025-01-04 21:38:45,656 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_3e9f0bce1c3a62e7f00b.table.json with policy now +2025-01-04 21:38:45,656 DEBUG SenderThread:2028802 [sender.py:send():379] send: history +2025-01-04 21:38:45,657 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:38:45,657 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:38:45,842 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:45,855 INFO wandb-upload_1:2028802 [upload_job.py:push():130] Uploaded file /tmp/tmp6wgcj3uiwandb/27zqpmqz-media/table/eval/total_success_rate_ci_6_3e9f0bce1c3a62e7f00b.table.json +2025-01-04 21:38:46,103 INFO wandb-upload_2:2028802 [upload_job.py:push():130] Uploaded file /tmp/tmp6wgcj3uiwandb/00paugon-media/table/eval/total_success_rate_ci_6_3e9f0bce1c3a62e7f00b.table.json +2025-01-04 21:38:46,390 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/wandb-summary.json +2025-01-04 21:38:46,390 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:38:46,843 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:47,843 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:48,658 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:38:48,843 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:49,843 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:50,843 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:51,843 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:52,843 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:53,765 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:38:53,843 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:54,325 DEBUG SenderThread:2028802 [sender.py:send():379] send: exit +2025-01-04 21:38:54,325 INFO SenderThread:2028802 [sender.py:send_exit():586] handling exit code: 0 +2025-01-04 21:38:54,326 INFO SenderThread:2028802 [sender.py:send_exit():588] handling runtime: 280 +2025-01-04 21:38:54,327 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:38:54,327 INFO SenderThread:2028802 [sender.py:send_exit():594] send defer +2025-01-04 21:38:54,327 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:38:54,327 INFO HandlerThread:2028802 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-04 21:38:54,327 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: defer +2025-01-04 21:38:54,327 INFO SenderThread:2028802 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-04 21:38:54,327 INFO SenderThread:2028802 [sender.py:transition_state():614] send defer: 1 +2025-01-04 21:38:54,327 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:38:54,327 INFO HandlerThread:2028802 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-04 21:38:54,328 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: defer +2025-01-04 21:38:54,328 INFO SenderThread:2028802 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-04 21:38:54,328 INFO SenderThread:2028802 [sender.py:transition_state():614] send defer: 2 +2025-01-04 21:38:54,328 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:38:54,328 INFO HandlerThread:2028802 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-04 21:38:54,328 INFO HandlerThread:2028802 [system_monitor.py:finish():203] Stopping system monitor +2025-01-04 21:38:54,328 INFO HandlerThread:2028802 [interfaces.py:finish():200] Joined cpu monitor +2025-01-04 21:38:54,328 INFO HandlerThread:2028802 [interfaces.py:finish():200] Joined disk monitor +2025-01-04 21:38:54,328 DEBUG SystemMonitor:2028802 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-04 21:38:54,328 DEBUG SystemMonitor:2028802 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-04 21:38:54,380 INFO HandlerThread:2028802 [interfaces.py:finish():200] Joined gpu monitor +2025-01-04 21:38:54,380 INFO HandlerThread:2028802 [interfaces.py:finish():200] Joined memory monitor +2025-01-04 21:38:54,380 INFO HandlerThread:2028802 [interfaces.py:finish():200] Joined network monitor +2025-01-04 21:38:54,381 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: defer +2025-01-04 21:38:54,381 INFO SenderThread:2028802 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-04 21:38:54,381 INFO SenderThread:2028802 [sender.py:transition_state():614] send defer: 3 +2025-01-04 21:38:54,381 DEBUG SenderThread:2028802 [sender.py:send():379] send: stats +2025-01-04 21:38:54,381 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:38:54,381 INFO HandlerThread:2028802 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-04 21:38:54,382 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: defer +2025-01-04 21:38:54,382 INFO SenderThread:2028802 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-04 21:38:54,382 INFO SenderThread:2028802 [sender.py:transition_state():614] send defer: 4 +2025-01-04 21:38:54,382 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:38:54,382 INFO HandlerThread:2028802 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-04 21:38:54,393 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/wandb-summary.json +2025-01-04 21:38:56,394 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:39:01,126 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:39:01,126 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:39:01,126 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: keepalive +2025-01-04 21:39:01,127 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: defer +2025-01-04 21:39:01,127 INFO SenderThread:2028802 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-04 21:39:01,127 INFO SenderThread:2028802 [sender.py:transition_state():614] send defer: 5 +2025-01-04 21:39:01,127 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:39:01,127 DEBUG SenderThread:2028802 [sender.py:send():379] send: history +2025-01-04 21:39:01,128 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:39:01,128 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:39:01,128 INFO HandlerThread:2028802 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-04 21:39:01,128 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:39:01,129 DEBUG SenderThread:2028802 [sender.py:send():379] send: summary +2025-01-04 21:39:01,132 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:39:01,133 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: defer +2025-01-04 21:39:01,133 INFO SenderThread:2028802 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-04 21:39:01,133 INFO SenderThread:2028802 [sender.py:transition_state():614] send defer: 6 +2025-01-04 21:39:01,133 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:39:01,133 INFO HandlerThread:2028802 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-04 21:39:01,133 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: defer +2025-01-04 21:39:01,133 INFO SenderThread:2028802 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-04 21:39:01,136 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:39:01,256 INFO SenderThread:2028802 [sender.py:transition_state():614] send defer: 7 +2025-01-04 21:39:01,256 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:39:01,256 INFO HandlerThread:2028802 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-04 21:39:01,257 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: defer +2025-01-04 21:39:01,257 INFO SenderThread:2028802 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-04 21:39:01,327 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:39:01,397 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/config.yaml +2025-01-04 21:39:01,397 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/wandb-summary.json +2025-01-04 21:39:03,909 INFO SenderThread:2028802 [sender.py:transition_state():614] send defer: 8 +2025-01-04 21:39:03,909 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:39:03,909 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:39:03,910 INFO HandlerThread:2028802 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-04 21:39:03,910 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: defer +2025-01-04 21:39:03,910 INFO SenderThread:2028802 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-04 21:39:03,910 INFO SenderThread:2028802 [job_builder.py:build():440] Attempting to build job artifact +2025-01-04 21:39:03,912 INFO SenderThread:2028802 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-04 21:39:03,914 INFO SenderThread:2028802 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-04 21:39:03,926 INFO SenderThread:2028802 [sender.py:transition_state():614] send defer: 9 +2025-01-04 21:39:03,927 DEBUG SenderThread:2028802 [sender.py:send():379] send: artifact +2025-01-04 21:39:03,929 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:39:03,929 INFO HandlerThread:2028802 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-04 21:39:04,201 INFO SenderThread:2028802 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxOTExMTc3MQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTEzMDY5OQ==', 'versionIndex': 419}}} +2025-01-04 21:39:04,201 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: defer +2025-01-04 21:39:04,201 INFO SenderThread:2028802 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-04 21:39:04,201 INFO SenderThread:2028802 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-04 21:39:04,327 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:39:04,398 INFO SenderThread:2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:39:04,398 INFO SenderThread:2028802 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files +2025-01-04 21:39:04,398 INFO SenderThread:2028802 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/config.yaml config.yaml +2025-01-04 21:39:04,398 INFO SenderThread:2028802 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/wandb-summary.json wandb-summary.json +2025-01-04 21:39:04,399 INFO SenderThread:2028802 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/events.out.tfevents.1736044453.leonmkim-ROG-Strix-G15CS-G15CS.2028630.0 events.out.tfevents.1736044453.leonmkim-ROG-Strix-G15CS-G15CS.2028630.0 +2025-01-04 21:39:04,399 INFO SenderThread:2028802 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/requirements.txt requirements.txt +2025-01-04 21:39:04,399 INFO SenderThread:2028802 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/wandb-metadata.json wandb-metadata.json +2025-01-04 21:39:04,399 INFO SenderThread:2028802 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log output.log +2025-01-04 21:39:04,399 INFO SenderThread:2028802 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/diff.patch diff.patch +2025-01-04 21:39:04,402 INFO SenderThread:2028802 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-04 21:39:04,407 INFO SenderThread:2028802 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval/0_eval_0_16d80c9a1dc6763cf29a.mp4 media/videos/eval/0_eval_0_16d80c9a1dc6763cf29a.mp4 +2025-01-04 21:39:04,407 INFO SenderThread:2028802 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval/2_eval_2_4982ced87e18fd534ea7.mp4 media/videos/eval/2_eval_2_4982ced87e18fd534ea7.mp4 +2025-01-04 21:39:04,407 INFO SenderThread:2028802 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval/1_eval_1_c9d9568f97ca21d8f63a.mp4 media/videos/eval/1_eval_1_c9d9568f97ca21d8f63a.mp4 +2025-01-04 21:39:04,407 INFO SenderThread:2028802 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval/4_eval_4_06b29357871c19fff932.mp4 media/videos/eval/4_eval_4_06b29357871c19fff932.mp4 +2025-01-04 21:39:04,407 INFO SenderThread:2028802 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval/3_eval_3_e8c539eab0fc6faa4df8.mp4 media/videos/eval/3_eval_3_e8c539eab0fc6faa4df8.mp4 +2025-01-04 21:39:04,407 INFO SenderThread:2028802 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-04 21:39:04,407 INFO SenderThread:2028802 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/table/eval/total_success_rate_ci_6_3e9f0bce1c3a62e7f00b.table.json media/table/eval/total_success_rate_ci_6_3e9f0bce1c3a62e7f00b.table.json +2025-01-04 21:39:04,407 INFO SenderThread:2028802 [sender.py:transition_state():614] send defer: 10 +2025-01-04 21:39:04,407 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:39:04,408 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:39:04,408 INFO HandlerThread:2028802 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-04 21:39:04,408 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: defer +2025-01-04 21:39:04,408 INFO SenderThread:2028802 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-04 21:39:04,408 INFO SenderThread:2028802 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 21:39:04,586 INFO wandb-upload_1:2028802 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/config.yaml +2025-01-04 21:39:04,591 INFO wandb-upload_0:2028802 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/wandb-summary.json +2025-01-04 21:39:04,634 INFO wandb-upload_3:2028802 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/requirements.txt +2025-01-04 21:39:04,646 INFO wandb-upload_4:2028802 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:39:04,663 INFO wandb-upload_2:2028802 [upload_job.py:push():130] Uploaded file /tmp/tmp6wgcj3uiwandb/0j38phy6-events.out.tfevents.1736044453.leonmkim-ROG-Strix-G15CS-G15CS.2028630.0 +2025-01-04 21:39:04,863 INFO Thread-11 (_thread_body):2028802 [sender.py:transition_state():614] send defer: 11 +2025-01-04 21:39:04,863 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:39:04,863 INFO HandlerThread:2028802 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-04 21:39:04,863 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: defer +2025-01-04 21:39:04,863 INFO SenderThread:2028802 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-04 21:39:04,863 INFO SenderThread:2028802 [file_pusher.py:join():175] waiting for file pusher +2025-01-04 21:39:04,865 INFO SenderThread:2028802 [sender.py:transition_state():614] send defer: 12 +2025-01-04 21:39:04,865 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:39:04,865 INFO HandlerThread:2028802 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-04 21:39:04,865 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: defer +2025-01-04 21:39:04,865 INFO SenderThread:2028802 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-04 21:39:04,865 INFO SenderThread:2028802 [file_stream.py:finish():601] file stream finish called +2025-01-04 21:39:04,936 INFO SenderThread:2028802 [file_stream.py:finish():605] file stream finish is done +2025-01-04 21:39:04,936 INFO SenderThread:2028802 [sender.py:transition_state():614] send defer: 13 +2025-01-04 21:39:04,936 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:39:04,936 INFO HandlerThread:2028802 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-04 21:39:04,936 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: defer +2025-01-04 21:39:04,936 INFO SenderThread:2028802 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-04 21:39:04,936 INFO SenderThread:2028802 [sender.py:transition_state():614] send defer: 14 +2025-01-04 21:39:04,936 DEBUG SenderThread:2028802 [sender.py:send():379] send: final +2025-01-04 21:39:04,936 DEBUG SenderThread:2028802 [sender.py:send():379] send: footer +2025-01-04 21:39:04,936 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:39:04,937 INFO HandlerThread:2028802 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-04 21:39:04,937 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: defer +2025-01-04 21:39:04,937 INFO SenderThread:2028802 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-04 21:39:04,937 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:39:04,937 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:39:04,938 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: server_info +2025-01-04 21:39:04,938 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: server_info +2025-01-04 21:39:04,938 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:39:04,939 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: get_summary +2025-01-04 21:39:04,940 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-04 21:39:04,940 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:39:05,001 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:39:05,001 INFO MainThread:2028802 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-04 21:39:05,001 INFO MainThread:2028802 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-04 21:39:05,002 INFO MainThread:2028802 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-04 21:39:05,002 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: shutdown +2025-01-04 21:39:05,002 INFO HandlerThread:2028802 [handler.py:finish():882] shutting down handler +2025-01-04 21:39:05,939 INFO WriterThread:2028802 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/run-nl3lmlc3.wandb +2025-01-04 21:39:06,001 INFO SenderThread:2028802 [sender.py:finish():1615] shutting down sender +2025-01-04 21:39:06,001 INFO SenderThread:2028802 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 21:39:06,001 INFO SenderThread:2028802 [file_pusher.py:join():175] waiting for file pusher diff --git a/nl3lmlc3/wandb/debug.log b/nl3lmlc3/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e55e46d502e9b36b76ca6130edbabd8d44d15ef1 --- /dev/null +++ b/nl3lmlc3/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-04 21:34:13,045 INFO MainThread:2028630 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-04 21:34:13,045 INFO MainThread:2028630 [wandb_setup.py:_flush():76] Configure stats pid to 2028630 +2025-01-04 21:34:13,045 INFO MainThread:2028630 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-04 21:34:13,046 INFO MainThread:2028630 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/settings +2025-01-04 21:34:13,046 INFO MainThread:2028630 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-04 21:34:13,046 INFO MainThread:2028630 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-04 21:34:13,046 INFO MainThread:2028630 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-04 21:34:13,046 INFO MainThread:2028630 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-04 21:34:13,046 INFO MainThread:2028630 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/logs/debug.log +2025-01-04 21:34:13,046 INFO MainThread:2028630 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/logs/debug-internal.log +2025-01-04 21:34:13,046 INFO MainThread:2028630 [wandb_init.py:init():569] calling init triggers +2025-01-04 21:34:13,046 INFO MainThread:2028630 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1926_1', 'wandb_notes': '1926_1_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3'} +2025-01-04 21:34:13,046 INFO MainThread:2028630 [wandb_init.py:init():619] starting backend +2025-01-04 21:34:13,046 INFO MainThread:2028630 [wandb_init.py:init():623] setting up manager +2025-01-04 21:34:13,051 INFO MainThread:2028630 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-04 21:34:13,052 INFO MainThread:2028630 [wandb_init.py:init():631] backend started and connected +2025-01-04 21:34:13,065 INFO MainThread:2028630 [wandb_init.py:init():720] updated telemetry +2025-01-04 21:34:13,072 INFO MainThread:2028630 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-04 21:34:13,325 INFO MainThread:2028630 [wandb_run.py:_on_init():2435] communicating current version +2025-01-04 21:34:13,392 INFO MainThread:2028630 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-04 21:34:13,392 INFO MainThread:2028630 [wandb_init.py:init():804] starting run threads in backend +2025-01-04 21:34:13,738 INFO MainThread:2028630 [wandb_run.py:_console_start():2413] atexit reg +2025-01-04 21:34:13,738 INFO MainThread:2028630 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-04 21:34:13,738 INFO MainThread:2028630 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-04 21:34:13,739 INFO MainThread:2028630 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-04 21:34:13,741 INFO MainThread:2028630 [wandb_init.py:init():847] run started, returning control to user process +2025-01-04 21:34:13,741 INFO MainThread:2028630 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/tb, True +2025-01-04 21:34:18,141 INFO MainThread:2028630 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-04 21:39:06,002 WARNING MsgRouterThr:2028630 [router.py:message_loop():77] message_loop has been closed diff --git a/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/code/FISH/eval_robot.py b/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/config.yaml b/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d66f00051915be3c2279747f68e6e3fa9fd6d86 --- /dev/null +++ b/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 1 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1926_1' +wandb_notes: + desc: null + value: 1926_1_restarted_5 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 1 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3 +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736044453 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/diff.patch b/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..b5be3e37ff49c316e2685f8bd605106c30e9bfb6 --- /dev/null +++ b/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/diff.patch @@ -0,0 +1,56 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 57a851f..27e7d93 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -138,12 +138,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' ++# wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1006_0' ++wandb_run_id: '1926_1' # seed 1 ++# wandb_run_id: '1930_0' # dataset shuffle seed 1 ++ ++ + + # RGBD+mask+contact(contact+DTC)+act history +-wandb_run_id: '1745_0' ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 71a5d1c..c7eadce 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -159,8 +159,10 @@ run_id_list = [ + ] + + run_id_list = [ +- # '1006_0', +- '1745_0' ++ # '1922_1', ++ # '1922_2', ++ '1926_1', ++ '1930_0', + ] + + checkpoint_epoch = 12000 +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index 6f97e26..285afc8 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,8 +330,8 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.005 +- self.move_to_rot_tolerance = 1.0 ++ self.move_to_trans_tolerance = 0.01 ++ self.move_to_rot_tolerance = 2.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) + self.move_to_target_timeout = 5.0 diff --git a/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json b/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json new file mode 100644 index 0000000000000000000000000000000000000000..eb10de88a713376e1c7301b69ab12adffa731488 --- /dev/null +++ b/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.7501049041247838, 1, 0.6051223873881857, 1, 0.6927025019737123, 1, 0.6332908421488255, 1]]} \ No newline at end of file diff --git a/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/table/eval/total_success_rate_ci_6_3e9f0bce1c3a62e7f00b.table.json b/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/table/eval/total_success_rate_ci_6_3e9f0bce1c3a62e7f00b.table.json new file mode 100644 index 0000000000000000000000000000000000000000..6abee9b47f8185f79c7b3fca149d1c267e50aed8 --- /dev/null +++ b/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/table/eval/total_success_rate_ci_6_3e9f0bce1c3a62e7f00b.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[26, 50, 0.52, 0.3683364391590232, 0.6744607984670763, 0.3784850090985141, 0.6496199369131302, 0.3970179483500451, 0.6353481560674814, 0.43777789549283874, 0.6135019324492902]]} \ No newline at end of file diff --git a/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval/0_eval_0_16d80c9a1dc6763cf29a.mp4 b/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval/0_eval_0_16d80c9a1dc6763cf29a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..606913388abb1b25dc3ff293f1ddfd57e7831126 --- /dev/null +++ b/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval/0_eval_0_16d80c9a1dc6763cf29a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16d80c9a1dc6763cf29a3b90accf6875237b314055606884f38bf6a1c8055147 +size 1450694 diff --git a/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval/1_eval_1_c9d9568f97ca21d8f63a.mp4 b/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval/1_eval_1_c9d9568f97ca21d8f63a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..51d3c54846475c1c4cefd67c60a98974e23437c6 --- /dev/null +++ b/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval/1_eval_1_c9d9568f97ca21d8f63a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9d9568f97ca21d8f63ad8b430332b1dddc7fc31a5f8686832b15d9d2c0adfc8 +size 1446649 diff --git a/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval/2_eval_2_4982ced87e18fd534ea7.mp4 b/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval/2_eval_2_4982ced87e18fd534ea7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39238d85726e63fd3422dc1960d752fad8ab679b --- /dev/null +++ b/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval/2_eval_2_4982ced87e18fd534ea7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4982ced87e18fd534ea7632a98a9e9397fbdc9ace6e94d61111a1cc8a488e7f5 +size 1464960 diff --git a/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval/3_eval_3_e8c539eab0fc6faa4df8.mp4 b/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval/3_eval_3_e8c539eab0fc6faa4df8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8a2720849b9d37e801068b5ee135e41366c4d3f9 --- /dev/null +++ b/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval/3_eval_3_e8c539eab0fc6faa4df8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8c539eab0fc6faa4df8dd4c58806f53068a10298a261b6f7a0016f5798a250f +size 1461798 diff --git a/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval/4_eval_4_06b29357871c19fff932.mp4 b/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval/4_eval_4_06b29357871c19fff932.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..515fbad906e22740a2026dcaadb9ee90f77854a8 --- /dev/null +++ b/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval/4_eval_4_06b29357871c19fff932.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06b29357871c19fff932800d727eabeeea718317e8e920b418420132fa692631 +size 1469723 diff --git a/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log b/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..aaa16074c574a68c102239ee28341ceaf1a15f26 --- /dev/null +++ b/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log @@ -0,0 +1,274 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736044458.141863]: resetting environment +[INFO] [1736044458.147356]: cleared current plan +[INFO] [1736044458.148281]: moving to home +[INFO] [1736044459.149358]: reached home +[INFO] [1736044459.150954]: reset action history +[INFO] [1736044461.861056]: environment reset +Starting episode 0 +[INFO] [1736044461.865088]: resetting environment +[INFO] [1736044461.868948]: cleared current plan +[INFO] [1736044461.869881]: moving to home +[INFO] [1736044462.872543]: reached home +[INFO] [1736044462.873890]: reset action history +[INFO] [1736044465.585008]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736044466.925046]: resetting environment +[INFO] [1736044466.930261]: cleared current plan +[INFO] [1736044466.931157]: moving to home +[INFO] [1736044467.933209]: reached home +[INFO] [1736044467.934446]: reset action history +[INFO] [1736044470.642509]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/episode_rosbags/episode_0_2025-01-04-21-34-30.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2028630.log +started roslaunch server http://158.130.50.37:38597/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2028919] +process[print_text-2]: started with pid [2028920] +process[rosbag_record-3]: started with pid [2028944] +started bagging! +For topic gripper_width: timestamp difference is -111061998 for nearest: 1736044473351530604 - target: 1736044473462592602 at idx 42 +[INFO] [1736044505.577257]: Storing episode... +[rosbag_record-3] killing on exit +[pub_text-1] killing on exit +[print_text-2] killing on exit +[INFO] [1736044506.327590]: Stored episode 1. +[INFO] [1736044506.327856]: Saving video... +[INFO] [1736044506.517501]: Video saved! +Starting episode 1 +[INFO] [1736044506.535984]: resetting environment +[INFO] [1736044506.544615]: cleared current plan +[INFO] [1736044506.548126]: moving to home +[INFO] [1736044510.150295]: reached home +[INFO] [1736044510.150756]: reset action history +[INFO] [1736044512.865494]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736044519.473524]: resetting environment +[INFO] [1736044519.477056]: cleared current plan +[INFO] [1736044519.477226]: moving to home +[INFO] [1736044520.478643]: reached home +[INFO] [1736044520.479045]: reset action history +[INFO] [1736044523.237842]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/episode_rosbags/episode_1_2025-01-04-21-35-23.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2028630.log +started roslaunch server http://158.130.50.37:43113/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2029068] +process[print_text-5]: started with pid [2029069] +process[rosbag_record-6]: started with pid [2029070] +started bagging! +For topic gripper_width: timestamp difference is 106511788 for nearest: 1736044535718211846 - target: 1736044535611700058 at idx 172 +[INFO] [1736044558.066133]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit[pub_text-4] killing on exit +[INFO] [1736044558.809154]: Stored episode 2. +[INFO] [1736044558.809446]: Saving video... +[INFO] [1736044559.111348]: Video saved! +Starting episode 2 +[INFO] [1736044559.148257]: resetting environment +[INFO] [1736044559.164792]: cleared current plan +[INFO] [1736044559.165727]: moving to home +[INFO] [1736044562.772362]: reached home +[INFO] [1736044562.773491]: reset action history +[INFO] [1736044565.499974]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736044567.708696]: resetting environment +[INFO] [1736044567.717695]: cleared current plan +[INFO] [1736044567.717866]: moving to home +[INFO] [1736044568.719258]: reached home +[INFO] [1736044568.719740]: reset action history +[INFO] [1736044571.427440]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/episode_rosbags/episode_2_2025-01-04-21-36-11.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2028630.log +started roslaunch server http://158.130.50.37:43921/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2029242] +process[print_text-8]: started with pid [2029243] +process[rosbag_record-9]: started with pid [2029244] +started bagging! +For topic gripper_width: timestamp difference is -198091765 for nearest: 1736044571851541500 - target: 1736044572049633265 at idx 4 +For topic gripper_width: timestamp difference is 110070680 for nearest: 1736044582218207095 - target: 1736044582108136415 at idx 138 +For topic gripper_width: timestamp difference is 111145365 for nearest: 1736044583518207896 - target: 1736044583407062531 at idx 155 +For topic gripper_width: timestamp difference is 112194566 for nearest: 1736044584818235187 - target: 1736044584706040621 at idx 171 +For topic gripper_width: timestamp difference is 120273956 for nearest: 1736044594718431600 - target: 1736044594598157644 at idx 249 +[INFO] [1736044606.421596]: Storing episode... +[print_text-8] killing on exit +[pub_text-7] killing on exit +[rosbag_record-9] killing on exit +[INFO] [1736044607.230666]: Stored episode 3. +[INFO] [1736044607.233107]: Saving video... +[WARN] [1736044607.776839]: Plan exhausted +[WARN] [1736044607.814833]: Plan exhausted +[INFO] [1736044607.763347]: Video saved! +Starting episode 3 +[INFO] [1736044607.799965]: resetting environment +[INFO] [1736044607.818241]: cleared current plan +[INFO] [1736044607.818424]: moving to home +[INFO] [1736044611.421424]: reached home +[INFO] [1736044611.426090]: reset action history +[INFO] [1736044614.335670]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736044620.858063]: resetting environment +[INFO] [1736044620.884083]: cleared current plan +[INFO] [1736044620.884345]: moving to home +[INFO] [1736044621.886734]: reached home +[INFO] [1736044621.887434]: reset action history +[INFO] [1736044624.617826]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/episode_rosbags/episode_3_2025-01-04-21-37-04.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2028630.log +started roslaunch server http://158.130.50.37:44791/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2029373] +process[print_text-11]: started with pid [2029374] +process[rosbag_record-12]: started with pid [2029398] +started bagging! +[INFO] [1736044659.482243]: Storing episode... +[print_text-11] killing on exit +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[INFO] [1736044660.311155]: Stored episode 4. +[INFO] [1736044660.317620]: Saving video... +[INFO] [1736044660.615342]: Video saved! +Starting episode 4 +[INFO] [1736044660.652201]: resetting environment +[INFO] [1736044660.701052]: cleared current plan +[INFO] [1736044660.704432]: moving to home +[INFO] [1736044664.318912]: reached home +[INFO] [1736044664.325075]: reset action history +[INFO] [1736044667.114882]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736044673.139777]: resetting environment +[INFO] [1736044673.181068]: cleared current plan +[INFO] [1736044673.186475]: moving to home +[INFO] [1736044674.193740]: reached home +[INFO] [1736044674.195291]: reset action history +[INFO] [1736044676.970149]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/episode_rosbags/episode_4_2025-01-04-21-37-56.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2028630.log +started roslaunch server http://158.130.50.37:39785/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2029548] +process[print_text-14]: started with pid [2029549] +process[rosbag_record-15]: started with pid [2029550] +started bagging! +For topic gripper_width: timestamp difference is 125712784 for nearest: 1736044684118213328 - target: 1736044683992500544 at idx 94 +For topic gripper_width: timestamp difference is 114173436 for nearest: 1736044710951545501 - target: 1736044710837372065 at idx 249 +[WARN] [1736044713.308602]: Plan exhausted +[WARN] [1736044713.353468]: Plan exhausted +[INFO] [1736044711.980546]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1736044712.876241]: Stored episode 5. +[INFO] [1736044712.884939]: Saving video... +[INFO] [1736044713.391086]: Video saved! +[INFO] [1736044713.447627]: resetting environment +[INFO] [1736044713.464611]: cleared current plan +[INFO] [1736044713.464885]: moving to home +[WARN] [1736044713.388549]: Plan exhausted +[WARN] [1736044713.428744]: Plan exhausted +[INFO] [1736044717.071205]: reached home +[INFO] [1736044717.074844]: reset action history +[INFO] [1736044719.919976]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/requirements.txt b/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/wandb-metadata.json b/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..f15c8725b6a5e13f8bbd80749c79fff596f9bfef --- /dev/null +++ b/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-05T02:34:13.414976", + "startedAt": "2025-01-05T02:34:13.042598", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "02cf07d901f3d8187a302cd51c9faa80321417ee" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3749.683, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 4499.919, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4497.128, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.97, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 688.5316734313965 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/wandb-summary.json b/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..e4fcc3ef10b488419e974332c4b7308ff2fa3458 --- /dev/null +++ b/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "16d80c9a1dc6763cf29a3b90accf6875237b314055606884f38bf6a1c8055147", "size": 1450694, "path": "media/videos/eval/0_eval_0_16d80c9a1dc6763cf29a.mp4"}, "global_step": 665, "_timestamp": 1736044724.4596667, "_runtime": 271.40705370903015, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "c9d9568f97ca21d8f63ad8b430332b1dddc7fc31a5f8686832b15d9d2c0adfc8", "size": 1446649, "path": "media/videos/eval/1_eval_1_c9d9568f97ca21d8f63a.mp4"}, "eval/num_success": 5.0, "episode": 50.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.7501049041748047, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.6051223874092102, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6927024722099304, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 0.6332908272743225, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "4982ced87e18fd534ea7632a98a9e9397fbdc9ace6e94d61111a1cc8a488e7f5", "size": 1464960, "path": "media/videos/eval/2_eval_2_4982ced87e18fd534ea7.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "e8c539eab0fc6faa4df8dd4c58806f53068a10298a261b6f7a0016f5798a250f", "size": 1461798, "path": "media/videos/eval/3_eval_3_e8c539eab0fc6faa4df8.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "06b29357871c19fff932800d727eabeeea718317e8e920b418420132fa692631", "size": 1469723, "path": "media/videos/eval/4_eval_4_06b29357871c19fff932.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "d8821c50291f0fbf979f12eb7aaa0b647dfd15fe05387335a9663efc94f7fc3c", "size": 330, "artifact_path": "wandb-client-artifact://6nfr911sts7qpr8cefum5zuiqhylm7kym03mehzu9eiulp88pqzk8uvsjo6gtqthxomphir2pavd53qpnuwq3wdelxsc4hik3vz0mik3q6hpfnykyclnmcqoasfg68c0/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://j3xv96myuhim388kvexehwbofax905dpb50l1ol8q5ott4d7zi04wvg16fhbhtl2bno6w0p2rwrvc8clrjaoxkkim5psjw8fmd1yju21ncpmyfq1e91278z1zy34icca:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 26, "total_num_episodes": 50, "total_success_rate": 0.52, "total_success_rate_lb_0.01": 0.3683364391590232, "total_success_rate_ub_0.01": 0.6744607984670763, "total_success_rate_lb_0.025": 0.3784850090985141, "total_success_rate_ub_0.025": 0.6496199369131302, "total_success_rate_lb_0.05": 0.3970179483500451, "total_success_rate_ub_0.05": 0.6353481560674814, "total_success_rate_lb_0.1": 0.43777789549283874, "total_success_rate_ub_0.1": 0.6135019324492902, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "3e9f0bce1c3a62e7f00b814c42b4e6353863993197a565b04c6acc9832960e0d", "size": 507, "artifact_path": "wandb-client-artifact://8td6p10havjpda96kcywww6ipv2q3h1q3hwvr089hs9x3r99vthvhnl6g36o26aveh0yxlqj3l2hanlvl7vpn7wz7p079x46jggwllkra5rk96u8j81ay38jsoheuw6q/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://z0zaau2o91d1l91sdimjxad41gnpcggycqs96wmgx44n2qesvak12v88xi7tzqa4kthypeo6nkk3z9nintrgvla8a93zvn84pfe9uue6xzsxdr5jj5s2wriumzakplah:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_3e9f0bce1c3a62e7f00b.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 280}, "eval/total_success_rate": 0.5199999809265137, "eval/total_success_rate_lb_0.01": 0.36833643913269043, "eval/total_success_rate_ub_0.01": 0.674460768699646, "eval/total_success_rate_lb_0.025": 0.3784850239753723, "eval/total_success_rate_ub_0.025": 0.6496199369430542, "eval/total_success_rate_lb_0.05": 0.3970179557800293, "eval/total_success_rate_ub_0.05": 0.6353481411933899, "eval/total_success_rate_lb_0.1": 0.43777790665626526, "eval/total_success_rate_ub_0.1": 0.6135019063949585} \ No newline at end of file diff --git a/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/logs/debug-internal.log b/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..ff20444d409656fb4672fc7d73c8dc64616e0625 --- /dev/null +++ b/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/logs/debug-internal.log @@ -0,0 +1,789 @@ +2025-01-04 21:34:13,052 INFO StreamThr :2028802 [internal.py:wandb_internal():85] W&B internal server running at pid: 2028802, started at: 2025-01-04 21:34:13.052159 +2025-01-04 21:34:13,054 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status +2025-01-04 21:34:13,065 INFO WriterThread:2028802 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/run-nl3lmlc3.wandb +2025-01-04 21:34:13,065 DEBUG SenderThread:2028802 [sender.py:send():379] send: header +2025-01-04 21:34:13,074 DEBUG SenderThread:2028802 [sender.py:send():379] send: run +2025-01-04 21:34:13,300 INFO SenderThread:2028802 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files +2025-01-04 21:34:13,300 INFO SenderThread:2028802 [sender.py:_start_run_threads():1188] run started: nl3lmlc3 with start time 1736044453.052613 +2025-01-04 21:34:13,325 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: check_version +2025-01-04 21:34:13,326 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: check_version +2025-01-04 21:34:13,396 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: run_start +2025-01-04 21:34:13,401 DEBUG HandlerThread:2028802 [system_info.py:__init__():26] System info init +2025-01-04 21:34:13,401 DEBUG HandlerThread:2028802 [system_info.py:__init__():41] System info init done +2025-01-04 21:34:13,401 INFO HandlerThread:2028802 [system_monitor.py:start():194] Starting system monitor +2025-01-04 21:34:13,401 INFO SystemMonitor:2028802 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-04 21:34:13,402 INFO HandlerThread:2028802 [system_monitor.py:probe():214] Collecting system info +2025-01-04 21:34:13,402 INFO SystemMonitor:2028802 [interfaces.py:start():188] Started cpu monitoring +2025-01-04 21:34:13,403 INFO SystemMonitor:2028802 [interfaces.py:start():188] Started disk monitoring +2025-01-04 21:34:13,404 INFO SystemMonitor:2028802 [interfaces.py:start():188] Started gpu monitoring +2025-01-04 21:34:13,404 INFO SystemMonitor:2028802 [interfaces.py:start():188] Started memory monitoring +2025-01-04 21:34:13,405 INFO SystemMonitor:2028802 [interfaces.py:start():188] Started network monitoring +2025-01-04 21:34:13,414 DEBUG HandlerThread:2028802 [system_info.py:probe():152] Probing system +2025-01-04 21:34:13,416 DEBUG HandlerThread:2028802 [system_info.py:_probe_git():137] Probing git +2025-01-04 21:34:13,421 DEBUG HandlerThread:2028802 [system_info.py:_probe_git():145] Probing git done +2025-01-04 21:34:13,421 DEBUG HandlerThread:2028802 [system_info.py:probe():200] Probing system done +2025-01-04 21:34:13,421 DEBUG HandlerThread:2028802 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-05T02:34:13.414976', 'startedAt': '2025-01-05T02:34:13.042598', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '02cf07d901f3d8187a302cd51c9faa80321417ee'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.683, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.919, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4497.128, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.97, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 688.5316734313965}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-04 21:34:13,421 INFO HandlerThread:2028802 [system_monitor.py:probe():224] Finished collecting system info +2025-01-04 21:34:13,421 INFO HandlerThread:2028802 [system_monitor.py:probe():227] Publishing system info +2025-01-04 21:34:13,421 DEBUG HandlerThread:2028802 [system_info.py:_save_code():44] Saving code +2025-01-04 21:34:13,427 DEBUG HandlerThread:2028802 [system_info.py:_save_code():67] Saving code done +2025-01-04 21:34:13,427 DEBUG HandlerThread:2028802 [system_info.py:_save_patches():84] Saving git patches +2025-01-04 21:34:13,488 DEBUG HandlerThread:2028802 [system_info.py:_save_patches():126] Saving git patches done +2025-01-04 21:34:13,489 INFO HandlerThread:2028802 [system_monitor.py:probe():229] Finished publishing system info +2025-01-04 21:34:13,492 DEBUG SenderThread:2028802 [sender.py:send():379] send: files +2025-01-04 21:34:13,492 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-04 21:34:13,492 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-04 21:34:13,492 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-04 21:34:13,737 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: python_packages +2025-01-04 21:34:13,737 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: python_packages +2025-01-04 21:34:13,738 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:34:13,738 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:34:13,740 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:13,741 INFO HandlerThread:2028802 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/tb" + save: true +} +control { + relay_id: "3a7195e3c3d043c5990cffbb2121e77d" +} +_info { + stream_id: "nl3lmlc3" +} + +2025-01-04 21:34:13,741 DEBUG HandlerThread:2028802 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-04 21:34:13,760 INFO wandb-upload_2:2028802 [upload_job.py:push():130] Uploaded file /tmp/tmp6wgcj3uiwandb/dxoriorl-diff.patch +2025-01-04 21:34:13,766 INFO wandb-upload_1:2028802 [upload_job.py:push():130] Uploaded file /tmp/tmp6wgcj3uiwandb/l6vnklxd-code/FISH/eval_robot.py +2025-01-04 21:34:13,772 INFO wandb-upload_0:2028802 [upload_job.py:push():130] Uploaded file /tmp/tmp6wgcj3uiwandb/qut0tp3p-wandb-metadata.json +2025-01-04 21:34:13,904 DEBUG SenderThread:2028802 [sender.py:send():379] send: telemetry +2025-01-04 21:34:13,904 DEBUG SenderThread:2028802 [sender.py:send():379] send: tbrecord +2025-01-04 21:34:13,904 DEBUG SenderThread:2028802 [sender.py:send():379] send: files +2025-01-04 21:34:13,904 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file events.out.tfevents.1736044453.leonmkim-ROG-Strix-G15CS-G15CS.2028630.0 with policy live +2025-01-04 21:34:14,107 INFO wandb-upload_2:2028802 [upload_job.py:push():130] Uploaded file /tmp/tmp6wgcj3uiwandb/tsbq0c5s-events.out.tfevents.1736044453.leonmkim-ROG-Strix-G15CS-G15CS.2028630.0 +2025-01-04 21:34:14,301 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/wandb-metadata.json +2025-01-04 21:34:14,302 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/requirements.txt +2025-01-04 21:34:14,302 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/code/FISH/eval_robot.py +2025-01-04 21:34:14,302 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:34:14,302 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/diff.patch +2025-01-04 21:34:14,302 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/events.out.tfevents.1736044453.leonmkim-ROG-Strix-G15CS-G15CS.2028630.0 +2025-01-04 21:34:14,302 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/code/FISH +2025-01-04 21:34:14,302 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/code +2025-01-04 21:34:14,738 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:15,738 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:16,301 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:34:16,738 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:17,739 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:18,141 DEBUG SenderThread:2028802 [sender.py:send():379] send: config +2025-01-04 21:34:18,141 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:34:18,739 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:19,739 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:20,302 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:34:20,739 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:21,739 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:22,740 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:23,738 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:34:23,739 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:24,303 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:34:24,740 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:25,740 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:26,740 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:27,740 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:28,304 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:34:28,738 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:34:28,738 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:34:28,740 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:28,889 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:34:29,740 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:30,305 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:34:30,740 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:31,741 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:32,305 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:34:32,741 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:33,743 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:34,738 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:34:34,743 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:35,743 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:36,307 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:34:36,743 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:37,743 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:38,743 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:39,739 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:34:39,743 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:40,743 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:41,743 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:42,744 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:43,738 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:34:43,739 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:34:43,781 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:44,744 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:44,800 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:34:45,309 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/config.yaml +2025-01-04 21:34:45,744 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:46,744 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:47,744 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:48,744 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:49,744 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:50,740 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:34:50,744 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:51,745 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:52,745 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:53,745 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:54,745 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:55,740 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:34:55,745 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:56,747 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:57,746 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:58,738 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:34:58,739 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:34:58,781 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:34:59,746 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:00,747 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:00,859 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:35:01,747 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:02,747 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:03,747 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:04,748 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:05,748 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:06,328 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:35:06,533 DEBUG SenderThread:2028802 [sender.py:send():379] send: files +2025-01-04 21:35:06,533 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_16d80c9a1dc6763cf29a.mp4 with policy now +2025-01-04 21:35:06,534 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:35:06,535 DEBUG SenderThread:2028802 [sender.py:send():379] send: history +2025-01-04 21:35:06,535 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:35:06,536 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:35:06,748 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:06,773 INFO wandb-upload_0:2028802 [upload_job.py:push():130] Uploaded file /tmp/tmp6wgcj3uiwandb/h48q6q0p-media/videos/eval/0_eval_0_16d80c9a1dc6763cf29a.mp4 +2025-01-04 21:35:07,315 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/wandb-summary.json +2025-01-04 21:35:07,315 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval/0_eval_0_16d80c9a1dc6763cf29a.mp4 +2025-01-04 21:35:07,315 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media +2025-01-04 21:35:07,315 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos +2025-01-04 21:35:07,315 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval +2025-01-04 21:35:07,579 INFO wandb-upload_2:2028802 [upload_job.py:push():130] Uploaded file /tmp/tmp6wgcj3uiwandb/2qhyr499-media/videos/eval/0_eval_0_16d80c9a1dc6763cf29a.mp4 +2025-01-04 21:35:07,748 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:08,315 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:35:08,748 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:09,749 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:10,749 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:11,749 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:12,193 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:35:12,316 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:35:12,749 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:13,405 DEBUG SystemMonitor:2028802 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-04 21:35:13,406 DEBUG SenderThread:2028802 [sender.py:send():379] send: stats +2025-01-04 21:35:13,739 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:35:13,739 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:35:13,749 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:14,317 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:35:14,749 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:15,749 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:16,749 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:17,750 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:17,843 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:35:18,749 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:19,750 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:20,319 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/events.out.tfevents.1736044453.leonmkim-ROG-Strix-G15CS-G15CS.2028630.0 +2025-01-04 21:35:20,319 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:35:20,554 INFO wandb-upload_0:2028802 [upload_job.py:push():130] Uploaded file /tmp/tmp6wgcj3uiwandb/j5dil0ce-events.out.tfevents.1736044453.leonmkim-ROG-Strix-G15CS-G15CS.2028630.0 +2025-01-04 21:35:20,750 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:21,750 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:22,320 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:35:22,751 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:23,238 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:35:23,754 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:24,321 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:35:24,751 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:25,751 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:26,321 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:35:26,752 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:27,752 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:28,739 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:35:28,739 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:35:28,785 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:28,896 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:35:29,752 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:30,752 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:31,752 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:32,752 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:33,752 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:34,749 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:35:34,752 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:35,765 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:36,753 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:37,753 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:38,326 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:35:38,753 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:39,749 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:35:39,753 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:40,754 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:41,754 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:42,754 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:43,406 DEBUG SenderThread:2028802 [sender.py:send():379] send: stats +2025-01-04 21:35:43,739 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:35:43,739 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:35:43,781 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:44,754 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:44,860 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:35:45,754 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:46,754 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:47,754 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:48,754 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:49,755 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:50,749 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:35:50,755 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:51,755 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:52,755 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:53,755 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:54,755 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:55,749 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:35:55,755 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:56,755 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:57,755 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:58,739 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:35:58,739 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:35:58,759 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:35:59,153 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:35:59,153 DEBUG SenderThread:2028802 [sender.py:send():379] send: files +2025-01-04 21:35:59,154 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_c9d9568f97ca21d8f63a.mp4 with policy now +2025-01-04 21:35:59,154 DEBUG SenderThread:2028802 [sender.py:send():379] send: history +2025-01-04 21:35:59,154 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:35:59,160 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:35:59,332 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/wandb-summary.json +2025-01-04 21:35:59,332 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval/1_eval_1_c9d9568f97ca21d8f63a.mp4 +2025-01-04 21:35:59,333 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval +2025-01-04 21:35:59,386 INFO wandb-upload_2:2028802 [upload_job.py:push():130] Uploaded file /tmp/tmp6wgcj3uiwandb/5v7yfz83-media/videos/eval/1_eval_1_c9d9568f97ca21d8f63a.mp4 +2025-01-04 21:35:59,637 INFO wandb-upload_0:2028802 [upload_job.py:push():130] Uploaded file /tmp/tmp6wgcj3uiwandb/3cfe767r-media/videos/eval/1_eval_1_c9d9568f97ca21d8f63a.mp4 +2025-01-04 21:35:59,758 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:00,333 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:36:00,760 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:01,166 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:36:01,762 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:02,762 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:03,771 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:04,334 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:36:04,766 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:05,764 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:06,335 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:36:06,541 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:36:06,764 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:07,764 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:08,152 DEBUG SenderThread:2028802 [sender.py:send():379] send: history +2025-01-04 21:36:08,152 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:36:08,153 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:36:08,336 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/wandb-summary.json +2025-01-04 21:36:08,336 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/events.out.tfevents.1736044453.leonmkim-ROG-Strix-G15CS-G15CS.2028630.0 +2025-01-04 21:36:08,336 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:36:08,765 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:09,764 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:10,336 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:36:10,764 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:11,595 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:36:11,779 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:12,337 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:36:12,772 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:13,407 DEBUG SenderThread:2028802 [sender.py:send():379] send: stats +2025-01-04 21:36:13,739 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:36:13,739 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:36:13,781 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:14,338 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:36:14,772 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:15,773 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:16,773 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:16,898 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:36:17,773 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:18,773 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:19,773 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:20,773 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:21,773 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:22,221 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:36:22,773 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:23,774 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:24,341 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:36:24,774 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:25,774 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:26,342 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:36:26,774 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:27,774 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:27,865 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:36:28,739 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:36:28,739 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:36:28,781 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:29,774 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:30,774 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:31,775 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:32,776 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:32,895 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:36:33,776 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:34,775 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:35,776 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:36,345 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:36:36,776 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:37,778 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:38,752 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:36:38,777 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:39,777 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:40,777 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:41,777 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:42,778 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:43,407 DEBUG SenderThread:2028802 [sender.py:send():379] send: stats +2025-01-04 21:36:43,739 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:36:43,740 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:36:43,781 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:43,896 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:36:44,778 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:45,778 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:46,781 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:47,781 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:47,797 DEBUG SenderThread:2028802 [sender.py:send():379] send: files +2025-01-04 21:36:47,797 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_4982ced87e18fd534ea7.mp4 with policy now +2025-01-04 21:36:47,841 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:36:47,845 DEBUG SenderThread:2028802 [sender.py:send():379] send: history +2025-01-04 21:36:47,845 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:36:47,851 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:36:48,039 INFO wandb-upload_2:2028802 [upload_job.py:push():130] Uploaded file /tmp/tmp6wgcj3uiwandb/vgauh208-media/videos/eval/2_eval_2_4982ced87e18fd534ea7.mp4 +2025-01-04 21:36:48,349 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/wandb-summary.json +2025-01-04 21:36:48,349 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:36:48,349 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval/2_eval_2_4982ced87e18fd534ea7.mp4 +2025-01-04 21:36:48,350 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval +2025-01-04 21:36:48,618 INFO wandb-upload_0:2028802 [upload_job.py:push():130] Uploaded file /tmp/tmp6wgcj3uiwandb/j3osszjo-media/videos/eval/2_eval_2_4982ced87e18fd534ea7.mp4 +2025-01-04 21:36:48,790 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:49,758 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:36:49,785 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:50,350 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:36:50,782 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:51,785 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:52,350 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:36:52,790 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:53,793 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:54,787 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:55,377 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:36:55,787 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:56,352 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:36:56,792 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:57,791 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:58,744 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:36:58,745 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:36:58,795 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:36:59,796 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:00,798 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:00,827 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:37:00,993 DEBUG SenderThread:2028802 [sender.py:send():379] send: history +2025-01-04 21:37:00,993 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:37:00,994 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:37:01,353 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/wandb-summary.json +2025-01-04 21:37:01,354 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/events.out.tfevents.1736044453.leonmkim-ROG-Strix-G15CS-G15CS.2028630.0 +2025-01-04 21:37:01,575 INFO wandb-upload_2:2028802 [upload_job.py:push():130] Uploaded file /tmp/tmp6wgcj3uiwandb/u47fpcpp-events.out.tfevents.1736044453.leonmkim-ROG-Strix-G15CS-G15CS.2028630.0 +2025-01-04 21:37:01,794 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:02,354 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:37:02,799 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:03,798 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:04,355 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:37:04,796 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:05,796 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:05,896 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:37:06,355 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:37:06,796 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:07,796 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:08,797 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:09,797 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:10,797 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:11,756 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:37:11,797 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:12,797 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:13,408 DEBUG SenderThread:2028802 [sender.py:send():379] send: stats +2025-01-04 21:37:13,741 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:37:13,741 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:37:13,797 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:14,797 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:15,797 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:16,798 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:16,816 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:37:17,798 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:18,798 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:19,798 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:20,798 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:21,798 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:22,756 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:37:22,798 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:23,798 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:24,799 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:25,799 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:26,799 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:27,757 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:37:27,799 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:28,741 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:37:28,741 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:37:28,799 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:29,799 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:30,799 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:31,800 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:32,800 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:32,848 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:37:33,800 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:34,800 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:35,800 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:36,800 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:37,800 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:38,757 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:37:38,800 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:39,804 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:40,366 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:37:40,647 DEBUG SenderThread:2028802 [sender.py:send():379] send: files +2025-01-04 21:37:40,647 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_e8c539eab0fc6faa4df8.mp4 with policy now +2025-01-04 21:37:40,650 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:37:40,650 DEBUG SenderThread:2028802 [sender.py:send():379] send: history +2025-01-04 21:37:40,651 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:37:40,739 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:37:40,805 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:40,893 INFO wandb-upload_0:2028802 [upload_job.py:push():130] Uploaded file /tmp/tmp6wgcj3uiwandb/xc5rptpf-media/videos/eval/3_eval_3_e8c539eab0fc6faa4df8.mp4 +2025-01-04 21:37:41,367 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/wandb-summary.json +2025-01-04 21:37:41,367 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval/3_eval_3_e8c539eab0fc6faa4df8.mp4 +2025-01-04 21:37:41,367 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval +2025-01-04 21:37:41,660 INFO wandb-upload_2:2028802 [upload_job.py:push():130] Uploaded file /tmp/tmp6wgcj3uiwandb/chdq7vcb-media/videos/eval/3_eval_3_e8c539eab0fc6faa4df8.mp4 +2025-01-04 21:37:41,817 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:42,367 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:37:42,812 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:43,408 DEBUG SenderThread:2028802 [sender.py:send():379] send: stats +2025-01-04 21:37:43,748 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:37:43,748 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:37:43,814 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:43,894 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:37:44,822 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:45,813 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:46,369 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:37:46,813 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:47,814 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:48,369 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:37:48,815 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:49,762 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:37:49,817 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:50,818 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:51,820 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:52,821 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:53,371 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/events.out.tfevents.1736044453.leonmkim-ROG-Strix-G15CS-G15CS.2028630.0 +2025-01-04 21:37:53,820 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:54,056 DEBUG SenderThread:2028802 [sender.py:send():379] send: history +2025-01-04 21:37:54,056 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:37:54,058 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:37:54,371 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/wandb-summary.json +2025-01-04 21:37:54,372 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:37:54,817 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:55,237 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:37:55,820 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:56,372 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:37:56,821 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:57,819 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:58,373 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:37:58,742 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:37:58,742 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:37:58,819 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:37:59,819 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:00,819 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:00,871 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:38:01,820 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:02,820 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:03,820 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:04,820 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:05,820 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:06,157 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:38:06,376 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:38:06,820 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:07,820 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:08,820 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:09,821 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:10,821 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:11,760 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:38:11,821 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:12,821 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:13,409 DEBUG SenderThread:2028802 [sender.py:send():379] send: stats +2025-01-04 21:38:13,742 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:38:13,743 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:38:13,821 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:14,821 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:15,821 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:16,821 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:16,840 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:38:17,822 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:18,822 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:19,822 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:20,822 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:21,822 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:22,760 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:38:22,822 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:23,822 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:24,822 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:25,822 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:26,823 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:27,760 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:38:27,823 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:28,742 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:38:28,743 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:38:28,823 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:29,823 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:30,823 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:31,823 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:32,385 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:38:32,833 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:32,881 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:38:33,442 DEBUG SenderThread:2028802 [sender.py:send():379] send: files +2025-01-04 21:38:33,442 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_06b29357871c19fff932.mp4 with policy now +2025-01-04 21:38:33,445 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:38:33,446 DEBUG SenderThread:2028802 [sender.py:send():379] send: history +2025-01-04 21:38:33,446 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:38:33,448 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:38:33,666 INFO wandb-upload_0:2028802 [upload_job.py:push():130] Uploaded file /tmp/tmp6wgcj3uiwandb/vt6hrxwp-media/videos/eval/4_eval_4_06b29357871c19fff932.mp4 +2025-01-04 21:38:33,823 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:34,386 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/wandb-summary.json +2025-01-04 21:38:34,386 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:38:34,386 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval/4_eval_4_06b29357871c19fff932.mp4 +2025-01-04 21:38:34,387 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval +2025-01-04 21:38:34,658 INFO wandb-upload_2:2028802 [upload_job.py:push():130] Uploaded file /tmp/tmp6wgcj3uiwandb/52pb5k3q-media/videos/eval/4_eval_4_06b29357871c19fff932.mp4 +2025-01-04 21:38:34,830 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:35,827 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:36,386 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:38:36,834 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:37,828 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:38,117 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:38:38,387 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:38:38,836 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:39,840 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:40,841 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:41,846 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:42,388 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:38:42,848 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:43,417 DEBUG SenderThread:2028802 [sender.py:send():379] send: stats +2025-01-04 21:38:43,417 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:38:43,751 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: stop_status +2025-01-04 21:38:43,751 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: stop_status +2025-01-04 21:38:43,852 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:43,974 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 21:38:43,974 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 21:38:44,125 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:38:44,389 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/events.out.tfevents.1736044453.leonmkim-ROG-Strix-G15CS-G15CS.2028630.0 +2025-01-04 21:38:44,389 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:38:44,390 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-04 21:38:44,390 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media +2025-01-04 21:38:44,390 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/table/eval +2025-01-04 21:38:44,390 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/table +2025-01-04 21:38:44,478 INFO wandb-upload_0:2028802 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmprpart9pb +2025-01-04 21:38:44,561 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-04 21:38:44,570 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: partial_history +2025-01-04 21:38:44,628 INFO wandb-upload_1:2028802 [upload_job.py:push():130] Uploaded file /tmp/tmp6wgcj3uiwandb/mkln1ksu-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-04 21:38:44,642 INFO wandb-upload_2:2028802 [upload_job.py:push():130] Uploaded file /tmp/tmp6wgcj3uiwandb/vwjt6kxp-events.out.tfevents.1736044453.leonmkim-ROG-Strix-G15CS-G15CS.2028630.0 +2025-01-04 21:38:44,842 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:44,869 INFO SenderThread:2028802 [sender.py:send_request_log_artifact():1525] logged artifact run-nl3lmlc3-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE1OTQ1NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1NzUzNg==', 'latestArtifact': None}} +2025-01-04 21:38:44,869 DEBUG SenderThread:2028802 [sender.py:send():379] send: files +2025-01-04 21:38:44,869 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now +2025-01-04 21:38:44,870 DEBUG SenderThread:2028802 [sender.py:send():379] send: history +2025-01-04 21:38:44,870 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:38:44,870 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:38:44,870 DEBUG SenderThread:2028802 [sender.py:send():379] send: history +2025-01-04 21:38:44,871 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:38:44,874 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:38:44,874 DEBUG SenderThread:2028802 [sender.py:send():379] send: telemetry +2025-01-04 21:38:44,875 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:38:44,875 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:38:44,875 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:38:44,875 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:38:44,875 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:38:44,876 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:38:44,876 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:38:44,876 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:38:44,876 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:38:44,877 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:38:44,877 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:38:44,877 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:38:44,877 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:38:44,878 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:38:44,878 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:38:44,878 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:38:44,878 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:38:44,879 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:38:44,879 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:38:44,879 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:38:44,879 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:38:44,879 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:38:44,880 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: log_artifact +2025-01-04 21:38:45,108 INFO wandb-upload_0:2028802 [upload_job.py:push():130] Uploaded file /tmp/tmp6wgcj3uiwandb/6shgp16c-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-04 21:38:45,265 INFO wandb-upload_2:2028802 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpjzr7cqer +2025-01-04 21:38:45,390 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/wandb-summary.json +2025-01-04 21:38:45,390 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/events.out.tfevents.1736044453.leonmkim-ROG-Strix-G15CS-G15CS.2028630.0 +2025-01-04 21:38:45,390 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/table/eval/total_success_rate_ci_6_3e9f0bce1c3a62e7f00b.table.json +2025-01-04 21:38:45,390 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/table/eval +2025-01-04 21:38:45,656 INFO SenderThread:2028802 [sender.py:send_request_log_artifact():1525] logged artifact run-nl3lmlc3-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxOTE1OTQ2OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTg1NzUzNw==', 'latestArtifact': None}} +2025-01-04 21:38:45,656 DEBUG SenderThread:2028802 [sender.py:send():379] send: files +2025-01-04 21:38:45,656 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_3e9f0bce1c3a62e7f00b.table.json with policy now +2025-01-04 21:38:45,656 DEBUG SenderThread:2028802 [sender.py:send():379] send: history +2025-01-04 21:38:45,657 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:38:45,657 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:38:45,842 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:45,855 INFO wandb-upload_1:2028802 [upload_job.py:push():130] Uploaded file /tmp/tmp6wgcj3uiwandb/27zqpmqz-media/table/eval/total_success_rate_ci_6_3e9f0bce1c3a62e7f00b.table.json +2025-01-04 21:38:46,103 INFO wandb-upload_2:2028802 [upload_job.py:push():130] Uploaded file /tmp/tmp6wgcj3uiwandb/00paugon-media/table/eval/total_success_rate_ci_6_3e9f0bce1c3a62e7f00b.table.json +2025-01-04 21:38:46,390 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/wandb-summary.json +2025-01-04 21:38:46,390 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:38:46,843 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:47,843 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:48,658 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:38:48,843 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:49,843 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:50,843 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:51,843 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:52,843 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:53,765 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:38:53,843 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:38:54,325 DEBUG SenderThread:2028802 [sender.py:send():379] send: exit +2025-01-04 21:38:54,325 INFO SenderThread:2028802 [sender.py:send_exit():586] handling exit code: 0 +2025-01-04 21:38:54,326 INFO SenderThread:2028802 [sender.py:send_exit():588] handling runtime: 280 +2025-01-04 21:38:54,327 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:38:54,327 INFO SenderThread:2028802 [sender.py:send_exit():594] send defer +2025-01-04 21:38:54,327 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:38:54,327 INFO HandlerThread:2028802 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-04 21:38:54,327 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: defer +2025-01-04 21:38:54,327 INFO SenderThread:2028802 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-04 21:38:54,327 INFO SenderThread:2028802 [sender.py:transition_state():614] send defer: 1 +2025-01-04 21:38:54,327 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:38:54,327 INFO HandlerThread:2028802 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-04 21:38:54,328 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: defer +2025-01-04 21:38:54,328 INFO SenderThread:2028802 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-04 21:38:54,328 INFO SenderThread:2028802 [sender.py:transition_state():614] send defer: 2 +2025-01-04 21:38:54,328 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:38:54,328 INFO HandlerThread:2028802 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-04 21:38:54,328 INFO HandlerThread:2028802 [system_monitor.py:finish():203] Stopping system monitor +2025-01-04 21:38:54,328 INFO HandlerThread:2028802 [interfaces.py:finish():200] Joined cpu monitor +2025-01-04 21:38:54,328 INFO HandlerThread:2028802 [interfaces.py:finish():200] Joined disk monitor +2025-01-04 21:38:54,328 DEBUG SystemMonitor:2028802 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-04 21:38:54,328 DEBUG SystemMonitor:2028802 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-04 21:38:54,380 INFO HandlerThread:2028802 [interfaces.py:finish():200] Joined gpu monitor +2025-01-04 21:38:54,380 INFO HandlerThread:2028802 [interfaces.py:finish():200] Joined memory monitor +2025-01-04 21:38:54,380 INFO HandlerThread:2028802 [interfaces.py:finish():200] Joined network monitor +2025-01-04 21:38:54,381 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: defer +2025-01-04 21:38:54,381 INFO SenderThread:2028802 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-04 21:38:54,381 INFO SenderThread:2028802 [sender.py:transition_state():614] send defer: 3 +2025-01-04 21:38:54,381 DEBUG SenderThread:2028802 [sender.py:send():379] send: stats +2025-01-04 21:38:54,381 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:38:54,381 INFO HandlerThread:2028802 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-04 21:38:54,382 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: defer +2025-01-04 21:38:54,382 INFO SenderThread:2028802 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-04 21:38:54,382 INFO SenderThread:2028802 [sender.py:transition_state():614] send defer: 4 +2025-01-04 21:38:54,382 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:38:54,382 INFO HandlerThread:2028802 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-04 21:38:54,393 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/wandb-summary.json +2025-01-04 21:38:56,394 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:39:01,126 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:39:01,126 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:39:01,126 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: keepalive +2025-01-04 21:39:01,127 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: defer +2025-01-04 21:39:01,127 INFO SenderThread:2028802 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-04 21:39:01,127 INFO SenderThread:2028802 [sender.py:transition_state():614] send defer: 5 +2025-01-04 21:39:01,127 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:39:01,127 DEBUG SenderThread:2028802 [sender.py:send():379] send: history +2025-01-04 21:39:01,128 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: summary_record +2025-01-04 21:39:01,128 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:39:01,128 INFO HandlerThread:2028802 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-04 21:39:01,128 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:39:01,129 DEBUG SenderThread:2028802 [sender.py:send():379] send: summary +2025-01-04 21:39:01,132 INFO SenderThread:2028802 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-04 21:39:01,133 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: defer +2025-01-04 21:39:01,133 INFO SenderThread:2028802 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-04 21:39:01,133 INFO SenderThread:2028802 [sender.py:transition_state():614] send defer: 6 +2025-01-04 21:39:01,133 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:39:01,133 INFO HandlerThread:2028802 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-04 21:39:01,133 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: defer +2025-01-04 21:39:01,133 INFO SenderThread:2028802 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-04 21:39:01,136 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: status_report +2025-01-04 21:39:01,256 INFO SenderThread:2028802 [sender.py:transition_state():614] send defer: 7 +2025-01-04 21:39:01,256 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:39:01,256 INFO HandlerThread:2028802 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-04 21:39:01,257 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: defer +2025-01-04 21:39:01,257 INFO SenderThread:2028802 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-04 21:39:01,327 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:39:01,397 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/config.yaml +2025-01-04 21:39:01,397 INFO Thread-12 :2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/wandb-summary.json +2025-01-04 21:39:03,909 INFO SenderThread:2028802 [sender.py:transition_state():614] send defer: 8 +2025-01-04 21:39:03,909 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:39:03,909 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:39:03,910 INFO HandlerThread:2028802 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-04 21:39:03,910 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: defer +2025-01-04 21:39:03,910 INFO SenderThread:2028802 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-04 21:39:03,910 INFO SenderThread:2028802 [job_builder.py:build():440] Attempting to build job artifact +2025-01-04 21:39:03,912 INFO SenderThread:2028802 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-04 21:39:03,914 INFO SenderThread:2028802 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-04 21:39:03,926 INFO SenderThread:2028802 [sender.py:transition_state():614] send defer: 9 +2025-01-04 21:39:03,927 DEBUG SenderThread:2028802 [sender.py:send():379] send: artifact +2025-01-04 21:39:03,929 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:39:03,929 INFO HandlerThread:2028802 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-04 21:39:04,201 INFO SenderThread:2028802 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxOTExMTc3MQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTEzMDY5OQ==', 'versionIndex': 419}}} +2025-01-04 21:39:04,201 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: defer +2025-01-04 21:39:04,201 INFO SenderThread:2028802 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-04 21:39:04,201 INFO SenderThread:2028802 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-04 21:39:04,327 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:39:04,398 INFO SenderThread:2028802 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:39:04,398 INFO SenderThread:2028802 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files +2025-01-04 21:39:04,398 INFO SenderThread:2028802 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/config.yaml config.yaml +2025-01-04 21:39:04,398 INFO SenderThread:2028802 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/wandb-summary.json wandb-summary.json +2025-01-04 21:39:04,399 INFO SenderThread:2028802 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/events.out.tfevents.1736044453.leonmkim-ROG-Strix-G15CS-G15CS.2028630.0 events.out.tfevents.1736044453.leonmkim-ROG-Strix-G15CS-G15CS.2028630.0 +2025-01-04 21:39:04,399 INFO SenderThread:2028802 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/requirements.txt requirements.txt +2025-01-04 21:39:04,399 INFO SenderThread:2028802 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/wandb-metadata.json wandb-metadata.json +2025-01-04 21:39:04,399 INFO SenderThread:2028802 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log output.log +2025-01-04 21:39:04,399 INFO SenderThread:2028802 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/diff.patch diff.patch +2025-01-04 21:39:04,402 INFO SenderThread:2028802 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-04 21:39:04,407 INFO SenderThread:2028802 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval/0_eval_0_16d80c9a1dc6763cf29a.mp4 media/videos/eval/0_eval_0_16d80c9a1dc6763cf29a.mp4 +2025-01-04 21:39:04,407 INFO SenderThread:2028802 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval/2_eval_2_4982ced87e18fd534ea7.mp4 media/videos/eval/2_eval_2_4982ced87e18fd534ea7.mp4 +2025-01-04 21:39:04,407 INFO SenderThread:2028802 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval/1_eval_1_c9d9568f97ca21d8f63a.mp4 media/videos/eval/1_eval_1_c9d9568f97ca21d8f63a.mp4 +2025-01-04 21:39:04,407 INFO SenderThread:2028802 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval/4_eval_4_06b29357871c19fff932.mp4 media/videos/eval/4_eval_4_06b29357871c19fff932.mp4 +2025-01-04 21:39:04,407 INFO SenderThread:2028802 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval/3_eval_3_e8c539eab0fc6faa4df8.mp4 media/videos/eval/3_eval_3_e8c539eab0fc6faa4df8.mp4 +2025-01-04 21:39:04,407 INFO SenderThread:2028802 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-04 21:39:04,407 INFO SenderThread:2028802 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/table/eval/total_success_rate_ci_6_3e9f0bce1c3a62e7f00b.table.json media/table/eval/total_success_rate_ci_6_3e9f0bce1c3a62e7f00b.table.json +2025-01-04 21:39:04,407 INFO SenderThread:2028802 [sender.py:transition_state():614] send defer: 10 +2025-01-04 21:39:04,407 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:39:04,408 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:39:04,408 INFO HandlerThread:2028802 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-04 21:39:04,408 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: defer +2025-01-04 21:39:04,408 INFO SenderThread:2028802 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-04 21:39:04,408 INFO SenderThread:2028802 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 21:39:04,586 INFO wandb-upload_1:2028802 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/config.yaml +2025-01-04 21:39:04,591 INFO wandb-upload_0:2028802 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/wandb-summary.json +2025-01-04 21:39:04,634 INFO wandb-upload_3:2028802 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/requirements.txt +2025-01-04 21:39:04,646 INFO wandb-upload_4:2028802 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/output.log +2025-01-04 21:39:04,663 INFO wandb-upload_2:2028802 [upload_job.py:push():130] Uploaded file /tmp/tmp6wgcj3uiwandb/0j38phy6-events.out.tfevents.1736044453.leonmkim-ROG-Strix-G15CS-G15CS.2028630.0 +2025-01-04 21:39:04,863 INFO Thread-11 (_thread_body):2028802 [sender.py:transition_state():614] send defer: 11 +2025-01-04 21:39:04,863 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:39:04,863 INFO HandlerThread:2028802 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-04 21:39:04,863 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: defer +2025-01-04 21:39:04,863 INFO SenderThread:2028802 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-04 21:39:04,863 INFO SenderThread:2028802 [file_pusher.py:join():175] waiting for file pusher +2025-01-04 21:39:04,865 INFO SenderThread:2028802 [sender.py:transition_state():614] send defer: 12 +2025-01-04 21:39:04,865 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:39:04,865 INFO HandlerThread:2028802 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-04 21:39:04,865 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: defer +2025-01-04 21:39:04,865 INFO SenderThread:2028802 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-04 21:39:04,865 INFO SenderThread:2028802 [file_stream.py:finish():601] file stream finish called +2025-01-04 21:39:04,936 INFO SenderThread:2028802 [file_stream.py:finish():605] file stream finish is done +2025-01-04 21:39:04,936 INFO SenderThread:2028802 [sender.py:transition_state():614] send defer: 13 +2025-01-04 21:39:04,936 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:39:04,936 INFO HandlerThread:2028802 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-04 21:39:04,936 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: defer +2025-01-04 21:39:04,936 INFO SenderThread:2028802 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-04 21:39:04,936 INFO SenderThread:2028802 [sender.py:transition_state():614] send defer: 14 +2025-01-04 21:39:04,936 DEBUG SenderThread:2028802 [sender.py:send():379] send: final +2025-01-04 21:39:04,936 DEBUG SenderThread:2028802 [sender.py:send():379] send: footer +2025-01-04 21:39:04,936 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: defer +2025-01-04 21:39:04,937 INFO HandlerThread:2028802 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-04 21:39:04,937 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: defer +2025-01-04 21:39:04,937 INFO SenderThread:2028802 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-04 21:39:04,937 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:39:04,937 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:39:04,938 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: server_info +2025-01-04 21:39:04,938 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: server_info +2025-01-04 21:39:04,938 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-04 21:39:04,939 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: get_summary +2025-01-04 21:39:04,940 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-04 21:39:04,940 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-04 21:39:05,001 DEBUG SenderThread:2028802 [sender.py:send_request():406] send_request: poll_exit +2025-01-04 21:39:05,001 INFO MainThread:2028802 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-04 21:39:05,001 INFO MainThread:2028802 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-04 21:39:05,002 INFO MainThread:2028802 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-04 21:39:05,002 DEBUG HandlerThread:2028802 [handler.py:handle_request():158] handle_request: shutdown +2025-01-04 21:39:05,002 INFO HandlerThread:2028802 [handler.py:finish():882] shutting down handler +2025-01-04 21:39:05,939 INFO WriterThread:2028802 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/run-nl3lmlc3.wandb +2025-01-04 21:39:06,001 INFO SenderThread:2028802 [sender.py:finish():1615] shutting down sender +2025-01-04 21:39:06,001 INFO SenderThread:2028802 [file_pusher.py:finish():169] shutting down file pusher +2025-01-04 21:39:06,001 INFO SenderThread:2028802 [file_pusher.py:join():175] waiting for file pusher diff --git a/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/logs/debug.log b/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e55e46d502e9b36b76ca6130edbabd8d44d15ef1 --- /dev/null +++ b/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-04 21:34:13,045 INFO MainThread:2028630 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-04 21:34:13,045 INFO MainThread:2028630 [wandb_setup.py:_flush():76] Configure stats pid to 2028630 +2025-01-04 21:34:13,045 INFO MainThread:2028630 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-04 21:34:13,046 INFO MainThread:2028630 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/settings +2025-01-04 21:34:13,046 INFO MainThread:2028630 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-04 21:34:13,046 INFO MainThread:2028630 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-04 21:34:13,046 INFO MainThread:2028630 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-04 21:34:13,046 INFO MainThread:2028630 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-04 21:34:13,046 INFO MainThread:2028630 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/logs/debug.log +2025-01-04 21:34:13,046 INFO MainThread:2028630 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/logs/debug-internal.log +2025-01-04 21:34:13,046 INFO MainThread:2028630 [wandb_init.py:init():569] calling init triggers +2025-01-04 21:34:13,046 INFO MainThread:2028630 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1926_1', 'wandb_notes': '1926_1_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3'} +2025-01-04 21:34:13,046 INFO MainThread:2028630 [wandb_init.py:init():619] starting backend +2025-01-04 21:34:13,046 INFO MainThread:2028630 [wandb_init.py:init():623] setting up manager +2025-01-04 21:34:13,051 INFO MainThread:2028630 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-04 21:34:13,052 INFO MainThread:2028630 [wandb_init.py:init():631] backend started and connected +2025-01-04 21:34:13,065 INFO MainThread:2028630 [wandb_init.py:init():720] updated telemetry +2025-01-04 21:34:13,072 INFO MainThread:2028630 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-04 21:34:13,325 INFO MainThread:2028630 [wandb_run.py:_on_init():2435] communicating current version +2025-01-04 21:34:13,392 INFO MainThread:2028630 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-04 21:34:13,392 INFO MainThread:2028630 [wandb_init.py:init():804] starting run threads in backend +2025-01-04 21:34:13,738 INFO MainThread:2028630 [wandb_run.py:_console_start():2413] atexit reg +2025-01-04 21:34:13,738 INFO MainThread:2028630 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-04 21:34:13,738 INFO MainThread:2028630 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-04 21:34:13,739 INFO MainThread:2028630 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-04 21:34:13,741 INFO MainThread:2028630 [wandb_init.py:init():847] run started, returning control to user process +2025-01-04 21:34:13,741 INFO MainThread:2028630 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/nl3lmlc3/tb, True +2025-01-04 21:34:18,141 INFO MainThread:2028630 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-04 21:39:06,002 WARNING MsgRouterThr:2028630 [router.py:message_loop():77] message_loop has been closed diff --git a/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/run-nl3lmlc3.wandb b/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/run-nl3lmlc3.wandb new file mode 100644 index 0000000000000000000000000000000000000000..6f3b185265da4e7a589cb4ad70760bb362a73b8f Binary files /dev/null and b/nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/run-nl3lmlc3.wandb differ diff --git a/p96n5ha2/episode_rosbags/episode_1_2025-01-03-22-59-41.bag b/p96n5ha2/episode_rosbags/episode_1_2025-01-03-22-59-41.bag new file mode 100644 index 0000000000000000000000000000000000000000..9017e15f7143ace54e23078782822304a943e10a --- /dev/null +++ b/p96n5ha2/episode_rosbags/episode_1_2025-01-03-22-59-41.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9dd0984159881d0cea4fd6b86cf1f1fcbfdb4cee8e51321fbb353d820522a7c3 +size 2473877656 diff --git a/p96n5ha2/episode_rosbags/episode_3_2025-01-03-23-01-30.bag b/p96n5ha2/episode_rosbags/episode_3_2025-01-03-23-01-30.bag new file mode 100644 index 0000000000000000000000000000000000000000..0a8b796040d46bfd7cfc4c74148d2b00eb978d7e --- /dev/null +++ b/p96n5ha2/episode_rosbags/episode_3_2025-01-03-23-01-30.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:071970575de9f9c06c80d8eb1a6909d9a92ef08a915ab3fadb082824fb56ef08 +size 2605530929 diff --git a/zc0axaef/episode_rosbags/episode_0_2025-01-04-21-29-24.bag b/zc0axaef/episode_rosbags/episode_0_2025-01-04-21-29-24.bag new file mode 100644 index 0000000000000000000000000000000000000000..1d5c51dc0f3787058d0da2d87504026567d16550 --- /dev/null +++ b/zc0axaef/episode_rosbags/episode_0_2025-01-04-21-29-24.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff3771f11af3888d427b8d63b5f51a77ec7f9908eef9cd4e5b081e49767b39b +size 2625626529 diff --git a/zc0axaef/episode_rosbags/episode_1_2025-01-04-21-30-17.bag b/zc0axaef/episode_rosbags/episode_1_2025-01-04-21-30-17.bag new file mode 100644 index 0000000000000000000000000000000000000000..a3fdc8ceecc132d7b8e02660be28522c4e109c29 --- /dev/null +++ b/zc0axaef/episode_rosbags/episode_1_2025-01-04-21-30-17.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55eb7ed9e4bc3a6823c266730f63c3486cf8a65fd07b23fdf4041ddef2724eb3 +size 2610073851 diff --git a/zc0axaef/episode_rosbags/episode_2_2025-01-04-21-31-11.bag b/zc0axaef/episode_rosbags/episode_2_2025-01-04-21-31-11.bag new file mode 100644 index 0000000000000000000000000000000000000000..7e36403fb6ab0c19b5960d935afc5aa5fa289706 --- /dev/null +++ b/zc0axaef/episode_rosbags/episode_2_2025-01-04-21-31-11.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f56d2343d0e67f571a427eefc958fe312879fc0d5be0d54d5c893ffa8359f2c3 +size 2609003542 diff --git a/zc0axaef/episode_rosbags/episode_3_2025-01-04-21-32-04.bag b/zc0axaef/episode_rosbags/episode_3_2025-01-04-21-32-04.bag new file mode 100644 index 0000000000000000000000000000000000000000..7ea5a0bba09a0c6e268cd341f975bb8c2b1355fe --- /dev/null +++ b/zc0axaef/episode_rosbags/episode_3_2025-01-04-21-32-04.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fc370b1d7f96633f65aaf3c15e38d07f9c9e2b94284d5c1dccdb9aada3fefcf +size 2608383424 diff --git a/zc0axaef/episode_rosbags/episode_4_2025-01-04-21-32-54.bag b/zc0axaef/episode_rosbags/episode_4_2025-01-04-21-32-54.bag new file mode 100644 index 0000000000000000000000000000000000000000..38e1fdbd3f1a5b323a24851ff00201c7bde5a0c1 --- /dev/null +++ b/zc0axaef/episode_rosbags/episode_4_2025-01-04-21-32-54.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:beb9a5ce8fca061e8cc7aa29eeee3b25d7e1b8cc6b37ed68c5df1f8c0eae8db1 +size 2602715693